diff options
Diffstat (limited to 'config-model/src')
169 files changed, 2033 insertions, 712 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index f324ceef5ab..2461fc64172 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -233,10 +233,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } } - // add cluster type? - // add cluster name? - public record StatePortInfo(String hostName, int portNumber, - String serviceName, String serviceType) + public record StatePortInfo(String hostName, int portNumber, Service service) {} public List<StatePortInfo> getStatePorts() { @@ -244,8 +241,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP for (HostResource modelHost : hostSystem().getHosts()) { String hostName = modelHost.getHostname(); for (Service modelService : modelHost.getServices()) { - String serviceName = modelService.getServiceName(); - String serviceType = modelService.getServiceType(); PortsMeta portsMeta = modelService.getPortsMeta(); for (int i = 0; i < portsMeta.getNumPorts(); i++) { int portNumber = modelService.getRelativePort(i); @@ -256,7 +251,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP if (tag.equals("http")) isHttp = true; } if (hasState && isHttp) { - result.add(new StatePortInfo(hostName, portNumber, serviceName, serviceType)); + result.add(new StatePortInfo(hostName, portNumber, modelService)); } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index e2c6b788b02..4ef591cda9f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -84,7 +84,7 @@ public class AdminModel extends ConfigModel { } TreeConfigProducer<AnyConfigProducer> parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext, properties.multitenant(), properties.configServerSpecs()); model.admin = domBuilder.build(modelContext.getDeployState(), parent, adminElement); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3c45588a054..c2d6adddeed 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -55,7 +55,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = List.of(); - private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; @@ -84,6 +83,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int contentLayerMetadataFeatureLevel = 0; private int persistenceThreadMaxFeedOpBatchSize = 1; private boolean logserverOtelCol = false; + private boolean symmetricPutAndActivateReplicaSelection = false; + private boolean enforceStrictlyIncreasingClusterStateVersions = false; + private boolean launchApplicationAthenzService = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -111,7 +113,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedNiceness() { return feedNiceness; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } - @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @@ -142,6 +143,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } @Override public boolean logserverOtelCol() { return logserverOtelCol; } + @Override public boolean symmetricPutAndActivateReplicaSelection() { return symmetricPutAndActivateReplicaSelection; } + @Override public boolean enforceStrictlyIncreasingClusterStateVersions() { return enforceStrictlyIncreasingClusterStateVersions; } + @Override public boolean launchApplicationAthenzService() { return launchApplicationAthenzService; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -276,11 +280,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setJvmOmitStackTraceInFastThrowOption(String value) { - this.jvmOmitStackTraceInFastThrowOption = value; - return this; - } - public TestProperties allowDisableMtls(boolean value) { this.allowDisableMtls = value; return this; @@ -382,6 +381,21 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setSymmetricPutAndActivateReplicaSelection(boolean symmetricReplicaSelection) { + this.symmetricPutAndActivateReplicaSelection = symmetricReplicaSelection; + return this; + } + + public TestProperties setEnforceStrictlyIncreasingClusterStateVersions(boolean enforce) { + this.enforceStrictlyIncreasingClusterStateVersions = enforce; + return this; + } + + public TestProperties setLaunchApplicationAthenzService(boolean launch) { + this.launchApplicationAthenzService = launch; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index befe57a97e4..dcaa7acf823 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -158,7 +158,7 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { - provisioned.add(cluster.id(), requested); + provisioned.add(cluster, requested); clusters.add(cluster); if (environment == Environment.dev && ! requested.isRequired()) { requested = requested.withLimits(requested.minResources().withNodes(1), @@ -305,8 +305,8 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public int compare(NodeResources a, NodeResources b) { - if (a.memoryGb() > b.memoryGb()) return 1; - if (a.memoryGb() < b.memoryGb()) return -1; + if (a.memoryGiB() > b.memoryGiB()) return 1; + if (a.memoryGiB() < b.memoryGiB()) return -1; if (a.diskGb() > b.diskGb()) return 1; if (a.diskGb() < b.diskGb()) return -1; return Double.compare(a.vcpu(), b.vcpu()); diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 82ed45028b3..ed1a4e98b49 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -73,7 +73,8 @@ public class RankProfile implements Cloneable { /** The resolved inherited profiles, or null when not resolved. */ private List<RankProfile> inherited; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); @@ -106,6 +107,7 @@ public class RankProfile implements Cloneable { /** The drop limit used to drop hits with rank score less than or equal to this value */ private double rankScoreDropLimit = -Double.MAX_VALUE; + private double secondPhaseRankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; private String inheritedSummaryFeaturesProfileName; @@ -141,6 +143,8 @@ public class RankProfile implements Cloneable { private Boolean strict; + private Boolean useSignificanceModel; + private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -216,6 +220,16 @@ public class RankProfile implements Cloneable { this.strict = strict; } + public void setUseSignificanceModel(Boolean useSignificanceModel) { + this.useSignificanceModel = useSignificanceModel; + } + + public boolean useSignificanceModel() { + if (useSignificanceModel != null) return useSignificanceModel; + return uniquelyInherited(RankProfile::useSignificanceModel, "use-model") + .orElse(false); // Disabled by default + } + /** * Adds a profile to those inherited by this. * The profile must belong to this schema (directly or by inheritance). @@ -294,20 +308,28 @@ public class RankProfile implements Cloneable { return false; } - public void setMatchPhaseSettings(MatchPhaseSettings settings) { + public void setMatchPhase(MatchPhaseSettings settings) { settings.checkValid(); - this.matchPhaseSettings = settings; + this.matchPhase = settings; } - public MatchPhaseSettings getMatchPhaseSettings() { - if (matchPhaseSettings != null) return matchPhaseSettings; - return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); + public MatchPhaseSettings getMatchPhase() { + if (matchPhase != null) return matchPhase; + return uniquelyInherited(RankProfile::getMatchPhase, "match phase settings").orElse(null); + } + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + public DiversitySettings getDiversity() { + if (diversity != null) return diversity; + return uniquelyInherited(RankProfile::getDiversity, "diversity settings").orElse(null); } /** Returns the uniquely determined property, where non-empty is defined as non-null */ private <T> Optional<T> uniquelyInherited(Function<RankProfile, T> propertyRetriever, String propertyDescription) { - return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); + return uniquelyInherited(propertyRetriever, Objects::nonNull, propertyDescription); } /** @@ -322,8 +344,8 @@ public class RankProfile implements Cloneable { Predicate<T> nonEmptyValueFilter, String propertyDescription) { Set<T> uniqueProperties = inherited().stream() - .map(p -> propertyRetriever.apply(p)) - .filter(p -> nonEmptyValueFilter.test(p)) + .map(propertyRetriever) + .filter(nonEmptyValueFilter) .collect(Collectors.toSet()); if (uniqueProperties.isEmpty()) return Optional.empty(); if (uniqueProperties.size() == 1) return uniqueProperties.stream().findAny(); @@ -482,7 +504,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getFirstPhase, "first-phase expression").orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -509,7 +531,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getSecondPhase, "second-phase expression").orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -529,7 +551,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getGlobalPhase() { if (globalPhaseRanking != null) return globalPhaseRanking; - return uniquelyInherited(p -> p.getGlobalPhase(), "global-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getGlobalPhase, "global-phase expression").orElse(null); } public void setGlobalPhaseRanking(String expression) { @@ -588,7 +610,7 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") + return uniquelyInherited(RankProfile::getSummaryFeatures, f -> ! f.isEmpty(), "summary features") .orElse(Set.of()); } @@ -605,13 +627,13 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") + return uniquelyInherited(RankProfile::getMatchFeatures, f -> ! f.isEmpty(), "match features") .orElse(Set.of()); } public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); - return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") + return uniquelyInherited(RankProfile::getHiddenMatchFeatures, f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); } @@ -649,7 +671,7 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") + return uniquelyInherited(RankProfile::getRankFeatures, f -> ! f.isEmpty(), "summary-features") .orElse(Set.of()); } @@ -680,7 +702,7 @@ public class RankProfile implements Cloneable { if (rankProperties.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); - var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") + var inheritedProperties = uniquelyInherited(RankProfile::getRankPropertyMap, m -> ! m.isEmpty(), "rank-properties") .orElse(Map.of()); if (rankProperties.isEmpty()) return inheritedProperties; @@ -722,21 +744,21 @@ public class RankProfile implements Cloneable { public int getRerankCount() { if (rerankCount >= 0) return rerankCount; - return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getRerankCount, c -> c >= 0, "rerank-count").orElse(-1); } public void setGlobalPhaseRerankCount(int count) { this.globalPhaseRerankCount = count; } public int getGlobalPhaseRerankCount() { if (globalPhaseRerankCount >= 0) return globalPhaseRerankCount; - return uniquelyInherited(p -> p.getGlobalPhaseRerankCount(), c -> c >= 0, "global-phase rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getGlobalPhaseRerankCount, c -> c >= 0, "global-phase rerank-count").orElse(-1); } public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } public int getNumThreadsPerSearch() { if (numThreadsPerSearch >= 0) return numThreadsPerSearch; - return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") + return uniquelyInherited(RankProfile::getNumThreadsPerSearch, n -> n >= 0, "num-threads-per-search") .orElse(-1); } @@ -744,14 +766,14 @@ public class RankProfile implements Cloneable { public int getMinHitsPerThread() { if (minHitsPerThread >= 0) return minHitsPerThread; - return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); + return uniquelyInherited(RankProfile::getMinHitsPerThread, n -> n >= 0, "min-hits-per-search").orElse(-1); } public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { if (numSearchPartitions >= 0) return numSearchPartitions; - return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); + return uniquelyInherited(RankProfile::getNumSearchPartitions, n -> n >= 0, "num-search-partitions").orElse(-1); } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } @@ -761,7 +783,7 @@ public class RankProfile implements Cloneable { public OptionalDouble getTermwiseLimit() { if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); - return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") + return uniquelyInherited(RankProfile::getTermwiseLimit, OptionalDouble::isPresent, "termwise-limit") .orElse(OptionalDouble.empty()); } @@ -769,21 +791,21 @@ public class RankProfile implements Cloneable { if (postFilterThreshold != null) { return OptionalDouble.of(postFilterThreshold); } - return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getPostFilterThreshold, OptionalDouble::isPresent, "post-filter-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getApproximateThreshold() { if (approximateThreshold != null) { return OptionalDouble.of(approximateThreshold); } - return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getApproximateThreshold, OptionalDouble::isPresent, "approximate-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getTargetHitsMaxAdjustmentFactor() { if (targetHitsMaxAdjustmentFactor != null) { return OptionalDouble.of(targetHitsMaxAdjustmentFactor); } - return uniquelyInherited(p -> p.getTargetHitsMaxAdjustmentFactor(), l -> l.isPresent(), "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getTargetHitsMaxAdjustmentFactor, OptionalDouble::isPresent, "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -793,24 +815,34 @@ public class RankProfile implements Cloneable { public Boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); + return uniquelyInherited(RankProfile::getIgnoreDefaultRankFeatures, "ignore-default-rank-features").orElse(false); } public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; - return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); + return uniquelyInherited(RankProfile::getKeepRankCount, c -> c >= 0, "keep-rank-count").orElse(-1); } public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } public double getRankScoreDropLimit() { if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; - return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") + return uniquelyInherited(RankProfile::getRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") .orElse(rankScoreDropLimit); } + public void setSecondPhaseRankScoreDropLimit(double limit) { this.secondPhaseRankScoreDropLimit = limit; } + + public double getSecondPhaseRankScoreDropLimit() { + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + return secondPhaseRankScoreDropLimit; + } + return uniquelyInherited(RankProfile::getSecondPhaseRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "second-phase rank-score-drop-limit") + .orElse(secondPhaseRankScoreDropLimit); + } + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -935,7 +967,7 @@ public class RankProfile implements Cloneable { } private boolean needToUpdateFunctionCache() { - if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; + if (inherited().stream().anyMatch(RankProfile::needToUpdateFunctionCache)) return true; return allFunctionsCached == null; } @@ -943,7 +975,7 @@ public class RankProfile implements Cloneable { /** Returns all filter fields in this profile and any profile it inherits. */ public Set<String> allFilterFields() { - Set<String> inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), + Set<String> inheritedFilterFields = uniquelyInherited(RankProfile::allFilterFields, fields -> ! fields.isEmpty(), "filter fields").orElse(Set.of()); if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); @@ -954,7 +986,7 @@ public class RankProfile implements Cloneable { } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { - if (expression.trim().length() == 0) + if (expression.trim().isEmpty()) throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { @@ -996,7 +1028,8 @@ public class RankProfile implements Cloneable { try { RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); - clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.matchPhase = this.matchPhase; // hmm? + clone.diversity = this.diversity; clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; @@ -1177,7 +1210,7 @@ public class RankProfile implements Cloneable { private Map<Reference, TensorType> featureTypes() { Map<Reference, TensorType> featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), + .collect(Collectors.toMap(Input::name, input -> input.type().tensorType())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); @@ -1494,15 +1527,9 @@ public class RankProfile implements Cloneable { private boolean ascending = false; private int maxHits = 0; // try to get this many hits before degrading the match phase private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. - private DiversitySettings diversity = null; private double evaluationPoint = 0.20; private double prePostFilterTippingPoint = 1.0; - public void setDiversity(DiversitySettings value) { - value.checkValid(); - diversity = value; - } - public void setAscending(boolean value) { ascending = value; } public void setAttribute(String value) { attribute = value; } public void setMaxHits(int value) { maxHits = value; } @@ -1514,7 +1541,6 @@ public class RankProfile implements Cloneable { public String getAttribute() { return attribute; } public int getMaxHits() { return maxHits; } public double getMaxFilterCoverage() { return maxFilterCoverage; } - public DiversitySettings getDiversity() { return diversity; } public double getEvaluationPoint() { return evaluationPoint; } public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } @@ -1678,7 +1704,7 @@ public class RankProfile implements Cloneable { } - public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + public record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { @Override public String toString() { return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; @@ -1699,7 +1725,7 @@ public class RankProfile implements Cloneable { } } - private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + private final List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..127d12594b4 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -721,7 +721,7 @@ public class Schema implements ImmutableSchema { "', but this schema does not exist"); // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later + // and require it to be explicit, so we have the option to support other possibilities later var parentDocument = owner.schemas().get(inherited.get()).getDocument(); if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) throw new IllegalArgumentException(this + " inherits '" + inherited.get() + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index c1b698df55f..3fb185e333d 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -106,15 +106,18 @@ public class IndexInfo extends Derived { } private static boolean isPositionField(ImmutableSDField field) { - return GeoPos.isAnyPos(field); + return (field != null) && GeoPos.isAnyPos(field); + } + private static boolean isMultivalueField(ImmutableSDField field) { + return (field != null) && field.getDataType().isMultivalue(); } @Override protected void derive(ImmutableSDField field, Schema schema) { - derive(field, schema, false); + derive(field, schema, null); } - protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { + protected void derive(ImmutableSDField field, Schema schema, ImmutableSDField parent) { if (field.getDataType().equals(DataType.PREDICATE)) { addIndexCommand(field, CMD_PREDICATE); Index index = field.getIndex(field.getName()); @@ -134,14 +137,13 @@ public class IndexInfo extends Derived { String name = e.getValue(); addIndexAlias(alias, name); } - boolean isPosition = isPositionField(field); if (field.usesStructOrMap()) { for (ImmutableSDField structField : field.getStructFields()) { - derive(structField, schema, isPosition); // Recursion + derive(structField, schema, field); // Recursion } } - if (isPosition) { + if (isPositionField(field)) { addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); } @@ -153,12 +155,12 @@ public class IndexInfo extends Derived { addIndexCommand(field, CMD_LOWERCASE); } - if (field.getDataType().isMultivalue()) { + if (isMultivalueField(field) || isMultivalueField(parent)) { addIndexCommand(field, CMD_MULTIVALUE); } Attribute attribute = field.getAttribute(); - if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { + if ((field.doesAttributing() || (attribute != null && !isPositionField(parent))) && !field.doesIndexing()) { addIndexCommand(field.getName(), CMD_ATTRIBUTE); if (attribute != null && attribute.isFastSearch()) addIndexCommand(field.getName(), CMD_FAST_SEARCH); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index b057624f055..15e5891a3e3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -159,6 +159,7 @@ public class RawRankProfile { private final boolean ignoreDefaultRankFeatures; private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final RankProfile.DiversitySettings diversitySettings; private final int rerankCount; private final int keepRankCount; private final int numThreadsPerSearch; @@ -169,6 +170,7 @@ public class RawRankProfile { private final OptionalDouble approximateThreshold; private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; + private final double secondPhaseRankScoreDropLimit; private final boolean sortBlueprintsByCost; private final boolean alwaysMarkPhraseExpensive; @@ -207,7 +209,8 @@ public class RawRankProfile { rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); - matchPhaseSettings = compiled.getMatchPhaseSettings(); + matchPhaseSettings = compiled.getMatchPhase(); + diversitySettings = compiled.getDiversity(); numThreadsPerSearch = compiled.getNumThreadsPerSearch(); minHitsPerThread = compiled.getMinHitsPerThread(); numSearchPartitions = compiled.getNumSearchPartitions(); @@ -219,6 +222,7 @@ public class RawRankProfile { targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); + secondPhaseRankScoreDropLimit = compiled.getSecondPhaseRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(compiled.getRankProperties()); @@ -486,13 +490,12 @@ public class RawRankProfile { properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); - RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); - if (diversitySettings != null) { - properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); - } + } + if (diversitySettings != null) { + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } if (rerankCount > -1) { properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); @@ -506,6 +509,9 @@ public class RawRankProfile { if (rankScoreDropLimit > -Double.MAX_VALUE) { properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + properties.add(new Pair<>("vespa.hitcollector.secondphase.rankscoredroplimit", secondPhaseRankScoreDropLimit + "")); + } if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index f996b2624db..b91404be2dd 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -183,10 +183,12 @@ public final class SchemaInfo extends Derived { private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (RankProfileInfo rankProfile : rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder() + .name(rankProfile.name()) + .hasSummaryFeatures(rankProfile.hasSummaryFeatures()) + .hasRankFeatures(rankProfile.hasRankFeatures()) + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder() + .useModel(rankProfile.useSignificanceModel())); for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); @@ -226,6 +228,7 @@ public final class SchemaInfo extends Derived { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { @@ -233,11 +236,13 @@ public final class SchemaInfo extends Derived { this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); this.inputs = profile.inputs(); + useSignificanceModel = profile.useSignificanceModel(); } public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } + public boolean useSignificanceModel() { return useSignificanceModel; } public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 398897c6b78..73495c066b9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; -import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 9d68553fa80..33256fa8586 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -33,6 +33,8 @@ public class Matching implements Cloneable, Serializable { private Integer maxLength; /** Maximum number of occurrences for each term */ private Integer maxTermOccurrences; + /** Maximum number of characters in a token. */ + private Integer maxTokenLength; private String exactMatchTerminator = null; @@ -61,6 +63,8 @@ public class Matching implements Cloneable, Serializable { public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public Integer maxTermOccurrences() { return maxTermOccurrences; } public Matching maxTermOccurrences(int maxTermOccurrences) { this.maxTermOccurrences = maxTermOccurrences; return this; } + public Integer maxTokenLength() { return maxTokenLength; } + public Matching maxTokenLength(int maxTokenLength) { this.maxTokenLength = maxTokenLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } public MatchAlgorithm getAlgorithm() { return algorithm; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index f165141b16e..2483fa47667 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -46,7 +46,7 @@ import java.util.TreeMap; * * @author bratseth */ -public class SDField extends Field implements TypedKey, ImmutableSDField { +public class SDField extends Field implements ImmutableSDField { /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ private boolean indexStructureField = false; @@ -315,7 +315,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { supplyStructField.accept(field.getName(), field.getDataType()); } } - if ((subType == null) && (structFields.size() > 0)) { + if ((subType == null) && (!structFields.isEmpty())) { throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + this + " [" + getDataType() + getDataType().getClass() + "] with " + structFields.size() + " struct fields"); @@ -627,7 +627,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); - if (name == null || "".equals(name)) { + if (name == null || name.isEmpty()) { name = getName(); attribute.setName(name); } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java deleted file mode 100644 index 652d21d7f7d..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataType; - -/** - * Common interface for various typed key (or field definitions). - * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping - * - * @author bratseth - */ -public interface TypedKey { - - String getName(); - - void setDataType(DataType type); - - DataType getDataType(); - -} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7659a1e6562..173eebe2a94 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -44,6 +44,7 @@ public class ConvertParsedFields { parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); parsed.getMaxTermOccurrences().ifPresent(maxTermOccurrences -> field.getMatching().maxTermOccurrences(maxTermOccurrences)); + parsed.getMaxTokenLength().ifPresent(maxTokenLength -> field.getMatching().maxTokenLength(maxTokenLength)); parsed.getMatchAlgorithm().ifPresent (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); parsed.getExactTerminator().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 5ccbb7b19a4..ff78a4a3b60 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -38,7 +38,8 @@ public class ConvertParsedRanking { for (String name : parsed.getInherited()) profile.inherit(name); - parsed.isStrict().ifPresent(value -> profile.setStrict(value)); + parsed.isStrict().ifPresent(profile::setStrict); + parsed.isUseSignificanceModel().ifPresent(profile::setUseSignificanceModel); for (var constant : parsed.getConstants().values()) profile.add(constant); @@ -57,39 +58,26 @@ public class ConvertParsedRanking { profile.addFunction(name, parameters, expression, inline); } - parsed.getRankScoreDropLimit().ifPresent - (value -> profile.setRankScoreDropLimit(value)); - parsed.getTermwiseLimit().ifPresent - (value -> profile.setTermwiseLimit(value)); - parsed.getPostFilterThreshold().ifPresent - (value -> profile.setPostFilterThreshold(value)); - parsed.getApproximateThreshold().ifPresent - (value -> profile.setApproximateThreshold(value)); - parsed.getTargetHitsMaxAdjustmentFactor().ifPresent - (value -> profile.setTargetHitsMaxAdjustmentFactor(value)); - parsed.getKeepRankCount().ifPresent - (value -> profile.setKeepRankCount(value)); - parsed.getMinHitsPerThread().ifPresent - (value -> profile.setMinHitsPerThread(value)); - parsed.getNumSearchPartitions().ifPresent - (value -> profile.setNumSearchPartitions(value)); - parsed.getNumThreadsPerSearch().ifPresent - (value -> profile.setNumThreadsPerSearch(value)); - parsed.getReRankCount().ifPresent - (value -> profile.setRerankCount(value)); - - parsed.getMatchPhaseSettings().ifPresent - (value -> profile.setMatchPhaseSettings(value)); - - parsed.getFirstPhaseExpression().ifPresent - (value -> profile.setFirstPhaseRanking(value)); - parsed.getSecondPhaseExpression().ifPresent - (value -> profile.setSecondPhaseRanking(value)); - - parsed.getGlobalPhaseExpression().ifPresent - (value -> profile.setGlobalPhaseRanking(value)); - parsed.getGlobalPhaseRerankCount().ifPresent - (value -> profile.setGlobalPhaseRerankCount(value)); + parsed.getRankScoreDropLimit().ifPresent(profile::setRankScoreDropLimit); + parsed.getSecondPhaseRankScoreDropLimit().ifPresent(profile::setSecondPhaseRankScoreDropLimit); + parsed.getTermwiseLimit().ifPresent(profile::setTermwiseLimit); + parsed.getPostFilterThreshold().ifPresent(profile::setPostFilterThreshold); + parsed.getApproximateThreshold().ifPresent(profile::setApproximateThreshold); + parsed.getTargetHitsMaxAdjustmentFactor().ifPresent(profile::setTargetHitsMaxAdjustmentFactor); + parsed.getKeepRankCount().ifPresent(profile::setKeepRankCount); + parsed.getMinHitsPerThread().ifPresent(profile::setMinHitsPerThread); + parsed.getNumSearchPartitions().ifPresent(profile::setNumSearchPartitions); + parsed.getNumThreadsPerSearch().ifPresent(profile::setNumThreadsPerSearch); + parsed.getReRankCount().ifPresent(profile::setRerankCount); + + parsed.getMatchPhase().ifPresent(profile::setMatchPhase); + parsed.getDiversity().ifPresent(profile::setDiversity); + + parsed.getFirstPhaseExpression().ifPresent(profile::setFirstPhaseRanking); + parsed.getSecondPhaseExpression().ifPresent(profile::setSecondPhaseRanking); + + parsed.getGlobalPhaseExpression().ifPresent(profile::setGlobalPhaseRanking); + parsed.getGlobalPhaseRerankCount().ifPresent(profile::setGlobalPhaseRerankCount); for (var value : parsed.getMatchFeatures()) { profile.addMatchFeatures(value); @@ -101,10 +89,8 @@ public class ConvertParsedRanking { profile.addSummaryFeatures(value); } - parsed.getInheritedMatchFeatures().ifPresent - (value -> profile.setInheritedMatchFeatures(value)); - parsed.getInheritedSummaryFeatures().ifPresent - (value -> profile.setInheritedSummaryFeatures(value)); + parsed.getInheritedMatchFeatures().ifPresent(profile::setInheritedMatchFeatures); + parsed.getInheritedSummaryFeatures().ifPresent(profile::setInheritedSummaryFeatures); if (parsed.getIgnoreDefaultRankFeatures()) { profile.setIgnoreDefaultRankFeatures(true); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 789f7023aed..063962bf0c4 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -133,9 +133,9 @@ public class IntermediateCollection { var parser = new SchemaParser(stream, deployLogger, modelProperties); try { parser.rankProfile(schema); - } catch (ParseException pe) { + } catch (ParseException | TokenMgrException e) { throw new ParseException("Failed parsing rank-profile from '" + reader.getName() + "': " + - stream.formatException(Exceptions.toMessageString(pe))); + stream.formatException(Exceptions.toMessageString(e))); } } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from '" + reader.getName() + "': " + ex.getMessage()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index c7d1a215ce3..bac2c894283 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -23,6 +23,7 @@ public class ParsedMatchSettings { private Integer gramSize = null; private Integer maxLength = null; private Integer maxTermOccurrences = null; + private Integer maxTokenLength = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } @@ -31,6 +32,7 @@ public class ParsedMatchSettings { Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } Optional<Integer> getMaxLength() { return Optional.ofNullable(maxLength); } Optional<Integer> getMaxTermOccurrences() { return Optional.ofNullable(maxTermOccurrences); } + Optional<Integer> getMaxTokenLength() { return Optional.ofNullable(maxTokenLength); } // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } @@ -40,5 +42,6 @@ public class ParsedMatchSettings { void setGramSize(int value) { this.gramSize = value; } void setMaxLength(int value) { this.maxLength = value; } void setMaxTermOccurrences(int value) { this.maxTermOccurrences = value; } + void setMaxTokenLength(int value) { this.maxTokenLength = value; } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index fbbb0c7fe83..2a117a4af4b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.schema.parser; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.DiversitySettings; import com.yahoo.schema.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; @@ -26,6 +27,7 @@ class ParsedRankProfile extends ParsedBlock { private boolean ignoreDefaultRankFeatures = false; private Double rankScoreDropLimit = null; + private Double secondPhaseRankScoreDropLimit = null; private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; @@ -38,12 +40,14 @@ class ParsedRankProfile extends ParsedBlock { private Integer numSearchPartitions = null; private Integer numThreadsPerSearch = null; private Integer reRankCount = null; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; private String firstPhaseExpression = null; private String inheritedSummaryFeatures = null; private String inheritedMatchFeatures = null; private String secondPhaseExpression = null; private Boolean strict = null; + private Boolean useSignificanceModel = null; private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>(); @@ -63,6 +67,7 @@ class ParsedRankProfile extends ParsedBlock { boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } Optional<Double> getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } + Optional<Double> getSecondPhaseRankScoreDropLimit() { return Optional.ofNullable(this.secondPhaseRankScoreDropLimit); } Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } Optional<Double> getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } Optional<Double> getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } @@ -75,7 +80,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<Integer> getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } Optional<Integer> getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } Optional<Integer> getReRankCount() { return Optional.ofNullable(this.reRankCount); } - Optional<MatchPhaseSettings> getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } + Optional<MatchPhaseSettings> getMatchPhase() { return Optional.ofNullable(this.matchPhase); } + Optional<DiversitySettings> getDiversity() { return Optional.ofNullable(this.diversity); } Optional<String> getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } Optional<String> getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } List<ParsedRankFunction> getFunctions() { return List.copyOf(functions.values()); } @@ -96,6 +102,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } Optional<Boolean> isStrict() { return Optional.ofNullable(this.strict); } + Optional<Boolean> isUseSignificanceModel() { return Optional.ofNullable(this.useSignificanceModel); } + void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } @@ -168,9 +176,13 @@ class ParsedRankProfile extends ParsedBlock { this.keepRankCount = count; } - void setMatchPhaseSettings(MatchPhaseSettings settings) { - verifyThat(matchPhaseSettings == null, "already has match-phase"); - this.matchPhaseSettings = settings; + void setMatchPhase(MatchPhaseSettings settings) { + verifyThat(matchPhase == null, "already has match-phase"); + this.matchPhase = settings; + } + void setDiversity(DiversitySettings settings) { + verifyThat(diversity == null, "already has diversity"); + this.diversity = settings; } void setMinHitsPerThread(int minHits) { @@ -193,6 +205,11 @@ class ParsedRankProfile extends ParsedBlock { this.rankScoreDropLimit = limit; } + void setSecondPhaseRankScoreDropLimit(double limit) { + verifyThat(secondPhaseRankScoreDropLimit == null, "already has rank-score-drop-limit for second phase"); + this.secondPhaseRankScoreDropLimit = limit; + } + void setRerankCount(int count) { verifyThat(reRankCount == null, "already has rerank-count"); this.reRankCount = count; @@ -218,6 +235,10 @@ class ParsedRankProfile extends ParsedBlock { this.strict = strict; } + void setUseSignificanceModel(boolean useSignificanceModel) { + verifyThat(this.useSignificanceModel == null, "already has use-model"); + this.useSignificanceModel = useSignificanceModel; + } void setTermwiseLimit(double limit) { verifyThat(termwiseLimit == null, "already has termwise-limit"); this.termwiseLimit = limit; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java index 762279e3871..e66cd62caa8 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java @@ -32,7 +32,7 @@ public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor @Override public void process(boolean validate, boolean documentsOnly) { schema.allImportedFields() - .forEach(field -> setTransform(field)); + .forEach(this::setTransform); } private Stream<SummaryField> getSummaryFieldsForImportedField(ImmutableSDField importedField) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java index 767593b82d0..769f0c9de92 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.TensorDataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.schema.Schema; @@ -45,6 +46,8 @@ public class AttributesImplicitWord extends Processor { private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { // numeric types should not trigger exact-match query parsing if (field.getDataType().getPrimitiveType() instanceof NumericDataType) return false; + // Tensor type should not trigger exact-match query parsing + if (field.getDataType() instanceof TensorDataType) return false; return (! field.hasIndex() && !field.getAttributes().isEmpty() diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java index 5c06ce25184..a8e0f86de8c 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -23,8 +23,8 @@ public class DiversitySettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { - validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + if (rankProfile.getDiversity() != null) { + validate(rankProfile, rankProfile.getDiversity()); } } } @@ -32,6 +32,9 @@ public class DiversitySettingsValidator extends Processor { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); + if ((rankProfile.getMatchPhase() == null) && (rankProfile.getSecondPhaseRanking() == null)) { + throw new IllegalArgumentException("In schema '" + schema.getName() + "', rank-profile '" + rankProfile.name() + "': 'diversity' requires either 'match-phase' or 'second-phase' to be specified."); + } } private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index 056c37a9830..4313ceb4be1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -75,7 +76,11 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + var maxTokenLength = field.getMatching().maxTokenLength(); + if (maxTokenLength == null) { + maxTokenLength = AnnotatorConfig.getDefaultMaxTokenLength(); + } + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, maxTokenLength).convert(field.getIndexingScript())); } } @@ -85,8 +90,12 @@ public class ExactMatch extends Processor { private static class MyProvider extends TypedTransformProvider { - MyProvider(Schema schema) { + private int maxTokenLength; + + MyProvider(Schema schema, int maxTokenLength) + { super(ExactExpression.class, schema); + this.maxTokenLength = maxTokenLength; } @Override @@ -96,7 +105,7 @@ public class ExactMatch extends Processor { @Override protected Expression newTransform(DataType fieldType) { - Expression exp = new ExactExpression(); + Expression exp = new ExactExpression(maxTokenLength); if (fieldType instanceof CollectionDataType) { exp = new ForEachExpression(exp); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java index f3a8f7cee18..d29820e0d51 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -25,7 +25,7 @@ public class MatchPhaseSettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhase(); if (settings != null) { validateMatchPhaseSettings(rankProfile, settings); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java index 4feb065a90b..5a13267f507 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -18,6 +18,9 @@ import java.util.Optional; */ public class PagedAttributeValidator extends Processor { + private static final String disavantagesUrl = + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages"; + public PagedAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, @@ -43,6 +46,10 @@ public class PagedAttributeValidator extends Processor { if (!isSupportedType(attribute)) { fail(schema, field, "The 'paged' attribute setting is not supported for fast-rank tensor and predicate types"); } + if (attribute.getType() == Attribute.Type.TENSOR && attribute.hnswIndexParams().isPresent()) { + warn(schema.getName(), field.getName(), "The 'paged' attribute setting in combination with " + + "HNSW indexing is strongly discouraged, see " + disavantagesUrl + " for details"); + } } private boolean isSupportedType(Attribute attribute) { @@ -58,8 +65,4 @@ public class PagedAttributeValidator extends Processor { return true; } - private boolean isDenseTensorType(TensorType type) { - return type.dimensions().stream().allMatch(d -> d.isIndexed()); - } - } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index dd36bbb3b61..8b0446364ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -13,6 +13,7 @@ import com.yahoo.schema.document.RankType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.objects.FieldBase; import java.util.Iterator; import java.util.List; @@ -110,8 +111,8 @@ public abstract class Processor { List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { - RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); + for (Iterator<RankProfile.RankSetting> j = profile.declaredRankSettingIterator(); j.hasNext(); ) { + RankProfile.RankSetting setting = j.next(); if (setting.getType().equals(type)) { someRankSettings.add(setting); } @@ -128,7 +129,7 @@ public abstract class Processor { return new IllegalArgumentException(formatError(schemaName, fieldName, msg)); } - protected RuntimeException newProcessException(Schema schema, Field field, String msg) { + protected RuntimeException newProcessException(Schema schema, FieldBase field, String msg) { return newProcessException(schema.getName(), field.getName(), msg); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e29f683761f..3f23cbc9b2d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -64,12 +64,16 @@ public class TextMatch extends Processor { if (fieldMatching != null) { var maxLength = fieldMatching.maxLength(); if (maxLength != null) { - ret.setMaxTokenLength(maxLength); + ret.setMaxTokenizeLength(maxLength); } var maxTermOccurrences = fieldMatching.maxTermOccurrences(); if (maxTermOccurrences != null) { ret.setMaxTermOccurrences(maxTermOccurrences); } + var maxTokenLength = fieldMatching.maxTokenLength(); + if (maxTokenLength != null) { + ret.setMaxTokenLength(maxTokenLength); + } } return ret; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java index 8ccc8870419..8be7be02135 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -29,6 +30,10 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { protected final boolean requiresTransform(Expression exp) { if (exp instanceof OutputExpression) { String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + // Incomplete output expressions never require a transform. + return false; + } if (exp instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) @@ -42,7 +47,7 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { fieldType = field.getDataType(); } else if (exp instanceof SummaryExpression) { - Field field = schema.getSummaryField(fieldName); + SummaryField field = schema.getSummaryField(fieldName); if (field == null) { // Use document field if summary field is not found var sdField = schema.getConcreteField(fieldName); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 32807db8405..e48aa0eef7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -2,19 +2,15 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.RankProfile; import com.yahoo.schema.Schema; -import com.yahoo.searchlib.rankingexpression.Reference; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; + +import static java.util.logging.Level.WARNING; /** * A document summary definition - a list of summary fields. @@ -25,7 +21,7 @@ public class DocumentSummary extends FieldView { private boolean fromDisk = false; private boolean omitSummaryFeatures = false; - private List<String> inherited = new ArrayList<>(); + private final List<String> inherited = new ArrayList<>(); private final Schema owner; @@ -115,7 +111,7 @@ public class DocumentSummary extends FieldView { /** Returns the parent of this, if any */ public List<DocumentSummary> inherited() { - return inherited.stream().map(name -> owner.getSummary(name)).toList(); + return inherited.stream().map(owner::getSummary).toList(); } @Override @@ -126,13 +122,12 @@ public class DocumentSummary extends FieldView { public void validate(DeployLogger logger) { for (var inheritedName : inherited) { var inheritedSummary = owner.getSummary(inheritedName); - if (inheritedSummary == null) { - // TODO Vespa 9: Throw IllegalArgumentException instead - logger.logApplicationPackage(Level.WARNING, - this + " inherits '" + inheritedName + "' but this" + " is not present in " + owner); - } + // TODO: Throw when no one is doing this anymore + if (inheritedName.equals("default")) + logger.logApplicationPackage(WARNING, this + " inherits '" + inheritedName + "', which makes no sense. Remove this inheritance"); + else if (inheritedSummary == null ) + throw new IllegalArgumentException(this + " inherits '" + inheritedName + "', but this is not present in " + owner); } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java index edc25bd8246..7034365c40b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.Field; +import com.yahoo.vespa.objects.FieldBase; import java.io.Serializable; import java.util.Collection; @@ -14,7 +14,7 @@ import java.util.Map; public class FieldView implements Serializable { private final String name; - private final Map<String, Field> fields = new LinkedHashMap<>(); + private final Map<String, FieldBase> fields = new LinkedHashMap<>(); /** * Creates a view with a name @@ -24,8 +24,8 @@ public class FieldView implements Serializable { this.name = name; } public String getName() { return name; } - public Collection<Field> getFields() { return fields.values(); } - public Field get(String name) { return fields.get(name); } + public Collection<FieldBase> getFields() { return fields.values(); } + public FieldBase get(String name) { return fields.get(name); } public void remove(String name) { fields.remove(name); } /** @@ -34,7 +34,7 @@ public class FieldView implements Serializable { * @param field the field to add. * @return itself for chaining purposes. */ - public FieldView add(Field field) { + public FieldView add(FieldBase field) { if (fields.containsKey(field.getName())) { if ( ! fields.get(field.getName()).equals(field)) { throw new IllegalArgumentException( @@ -54,7 +54,7 @@ public class FieldView implements Serializable { * @return Itself for chaining. */ public FieldView add(FieldView view) { - for(Field field : view.getFields()) { + for(var field : view.getFields()) { add(field); } return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index d50d5e36134..e529779735f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -3,10 +3,13 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.schema.document.TypedKey; +import com.yahoo.vespa.objects.FieldBase; import java.io.Serializable; -import java.util.*; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -16,7 +19,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author bratseth */ -public class SummaryField extends Field implements Cloneable, TypedKey { +public class SummaryField extends FieldBase implements Cloneable { /** A source (field name). */ public static class Source implements Serializable { @@ -53,6 +56,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** The command used per field in vsmsummary */ private VsmCommand vsmCommand = VsmCommand.NONE; + private DataType dataType; /** * The data sources for this output summary field, in prioritized order @@ -62,7 +66,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set<Source> sources = new java.util.LinkedHashSet<>(); - private Set<String> destinations =new java.util.LinkedHashSet<>(); + private Set<String> destinations = new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ private boolean implicit = false; @@ -84,8 +88,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } public SummaryField(String name, DataType type, SummaryTransform transform) { - super(name, type); + super(name); this.transform=transform; + this.dataType = type; } public static SummaryField createWithUnresolvedType(String name) { @@ -106,6 +111,10 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public boolean hasUnresolvedType() { return unresolvedType; } + public DataType getDataType() { + return dataType; + } + public void setTransform(SummaryTransform transform) { this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { @@ -194,7 +203,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { if (merge.getTransform() != getTransform()) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different transforms"); - if (!merge.getDataType().equals(getDataType())) + if (!merge.dataType.equals(dataType)) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different types"); setImplicit(false); @@ -250,7 +259,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns a string which aids locating this field in the source search definition */ public String toLocateString() { - return "summary " + getName() + " type " + toLowerCase(getDataType().getName()) + " in " + getDestinationString(); + return "summary " + getName() + " type " + toLowerCase(dataType.getName()) + " in " + getDestinationString(); } @Override @@ -290,9 +299,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public void setResolvedDataType(DataType type) { this.dataType = type; - if (!hasForcedId()) { - this.fieldId = calculateIdV7(null); - } unresolvedType = false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index d8149486b32..806f14da265 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -342,6 +342,7 @@ public abstract class AbstractService extends TreeConfigProducer<AnyConfigProduc return getServicePropertyString(key, null); } + @Override public String getServicePropertyString(String key, String defStr) { Object result = serviceProperties.get(key); return (result == null) ? defStr : result.toString(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a8085919a98..f87f1382ffb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -16,7 +16,7 @@ import java.util.Objects; public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> { // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy). - // Keep in sync with node-repository/ClusterModel. + // Keep in sync with node-repository/ClusterModel and startup scripts (go and bash). public static final double memoryOverheadGb = 0.7; private ConfigSentinel configSentinel = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index ad728d60c5a..12892cb12ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -1,23 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.container.Container; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. - * Only in use in hosted Vespa. */ public class LogserverContainer extends Container { public LogserverContainer(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "" + 0, 0, deployState); - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) - useDynamicPorts(); } @Override @@ -30,9 +25,4 @@ public class LogserverContainer extends Container { return ""; } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ca602b447fe..be38298279f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -97,11 +97,6 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - private void configureZooKeeperServer(boolean runStandaloneZooKeeper) { if (runStandaloneZooKeeper) ContainerModelBuilder.addReconfigurableZooKeeperServerComponents(this); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index c4702ba3543..8f262281edc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -10,7 +10,6 @@ import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; @@ -78,11 +77,6 @@ public class MetricsProxyContainer extends Container implements } @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - - @Override public int getWantedPort() { return BASEPORT; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java index fcd587622da..364048ff261 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.OpenTelemetryConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -17,10 +18,14 @@ import java.util.Optional; public class OpenTelemetryCollector extends AbstractService implements OpenTelemetryConfig.Producer { private final Zone zone; + private final ApplicationId applicationId; + private final boolean isHostedVespa; public OpenTelemetryCollector(TreeConfigProducer<?> parent) { super(parent, "otelcol"); this.zone = null; + this.applicationId = null; + this.isHostedVespa = false; setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -28,6 +33,8 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem public OpenTelemetryCollector(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "otelcol"); this.zone = deployState.zone(); + this.applicationId = deployState.getProperties().applicationId(); + this.isHostedVespa = deployState.isHosted(); setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -50,7 +57,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem @Override public void getConfig(OpenTelemetryConfig.Builder builder) { - var generator = new OpenTelemetryConfigGenerator(zone); + var generator = new OpenTelemetryConfigGenerator(zone, applicationId, isHostedVespa); AnyConfigProducer pp = this; AnyConfigProducer p = pp.getParent(); while (p != null && p != pp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 36eab6a04b3..9bac6e4553d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -1,16 +1,28 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.otel; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.Set; +import java.util.TreeSet; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -24,8 +36,14 @@ public class OpenTelemetryConfigGenerator { private final String cert_file; private final String key_file; private List<StatePortInfo> statePorts = new ArrayList<>(); + private final Zone zone; + private final ApplicationId applicationId; + private final boolean isHostedVespa; - OpenTelemetryConfigGenerator(Zone zone) { + OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId, boolean isHostedVespa) { + this.zone = zone; + this.applicationId = applicationId; + this.isHostedVespa = isHostedVespa; boolean isCd = true; boolean isPublic = true; if (zone != null) { @@ -81,12 +99,38 @@ public class OpenTelemetryConfigGenerator { if (useTls) addTls(g); { g.writeFieldName("labels"); + var dimVals = serviceAttributes(statePort.service()); + // these will be tagged as dimension-name/label-value + // attributes on all metrics from this /state/v1 port g.writeStartObject(); - g.writeStringField("service_type", statePort.serviceType()); + for (var entry : dimVals.entrySet()) { + if (entry.getValue() != null) { + g.writeStringField(entry.getKey(), entry.getValue()); + } + } + String ph = findParentHost(statePort.hostName()); + if (isHostedVespa && ph != null) { + g.writeStringField("parentHostname", ph); + } g.writeEndObject(); } g.writeEndObject(); } + // note: this pattern should match entire node name + static private final Pattern expectedNodeName1 = Pattern.compile("[a-z0-9]+-v6-[0-9]+[.].+"); + static private final Pattern expectedNodeName2 = Pattern.compile("[a-z]*[0-9]+[a-z][.].+"); + // matches the part we want to replace with just a dot + static private final Pattern replaceNodeName1 = Pattern.compile("-v6-[0-9]+[.]"); + static private final Pattern replaceNodeName2 = Pattern.compile("[a-z][.]"); + static String findParentHost(String nodeName) { + if (expectedNodeName1.matcher(nodeName).matches()) { + return replaceNodeName1.matcher(nodeName).replaceFirst("."); + } + if (expectedNodeName2.matcher(nodeName).matches()) { + return replaceNodeName2.matcher(nodeName).replaceFirst("."); + } + return null; + } private void addTls(JsonGenerator g) throws java.io.IOException { g.writeFieldName("tls"); g.writeStartObject(); @@ -123,6 +167,91 @@ public class OpenTelemetryConfigGenerator { } g.writeEndObject(); // file } + private void addProcessors(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("processors"); + g.writeStartObject(); + addResourceProcessor(g); + addRenameProcessor(g); + addFilterProcessor(g); + g.writeEndObject(); + } + private void addRenameProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metricstransform/rename"); + g.writeStartObject(); + g.writeFieldName("transforms"); + g.writeStartArray(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + if (! metric.name.equals(metric.outputName)) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + addRenameAction(g, oldName, newName); + } + } + g.writeEndArray(); + g.writeEndObject(); + } + private void addRenameAction(JsonGenerator g, String oldName, String newName) throws java.io.IOException { + g.writeStartObject(); + g.writeStringField("include", oldName); + g.writeStringField("action", "update"); + g.writeStringField("new_name", newName); + g.writeEndObject(); + } + private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("resource"); + g.writeStartObject(); + g.writeFieldName("attributes"); + g.writeStartArray(); + // common attributes for all metrics from all services; + // which application and which cloud/system/zone/environment + addAttributeInsert(g, PublicDimensions.ZONE, zoneAttr()); + addAttributeInsert(g, PublicDimensions.APPLICATION_ID, appIdAttr()); + addAttributeInsert(g, "system", systemAttr()); + addAttributeInsert(g, "tenantName", tenantAttr()); + addAttributeInsert(g, "applicationName", appNameAttr()); + addAttributeInsert(g, "instanceName", appInstanceAttr()); + addAttributeInsert(g, "cloud", cloudAttr()); + g.writeEndArray(); + g.writeEndObject(); + } + private void addAttributeInsert(JsonGenerator g, String key, String value) throws java.io.IOException { + if (value == null) return; + g.writeStartObject(); + g.writeStringField("key", key); + g.writeStringField("value", value); + g.writeStringField("action", "insert"); + g.writeEndObject(); + } + static private Set<String> wantedMetrics() { + Set<String> result = new TreeSet<>(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + result.add(oldName); + result.add(newName); + } + return result; + } + private void addFilterProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("filter/metricset"); + g.writeStartObject(); + g.writeFieldName("metrics"); + g.writeStartObject(); + g.writeFieldName("include"); + g.writeStartObject(); + g.writeStringField("match_type", "strict"); + g.writeFieldName("metric_names"); + g.writeStartArray(); + for (String metricName : wantedMetrics()) { + g.writeString(metricName); + } + g.writeEndArray(); + g.writeEndObject(); + g.writeEndObject(); + g.writeEndObject(); + } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); @@ -159,6 +288,9 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("metricstransform/rename"); + g.writeString("filter/metricset"); + g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); @@ -186,6 +318,7 @@ public class OpenTelemetryConfigGenerator { g.writeStartObject(); addReceivers(g); addExporters(g); + addProcessors(g); addServiceBlock(g); g.writeEndObject(); // root g.close(); @@ -203,4 +336,72 @@ public class OpenTelemetryConfigGenerator { List<String> referencedPaths() { return List.of(ca_file, cert_file, key_file); } + + private String zoneAttr() { + if (zone == null) return null; + return zone.environment().value() + "." + zone.region().value(); + } + private String appIdAttr() { + if (applicationId == null) return null; + return applicationId.toFullString(); + } + private String systemAttr() { + if (zone == null) return null; + return zone.system().value(); + } + private String tenantAttr() { + if (applicationId == null) return null; + return applicationId.tenant().value(); + } + private String appNameAttr() { + if (applicationId == null) return null; + return applicationId.application().value(); + } + private String appInstanceAttr() { + if (applicationId == null) return null; + return applicationId.instance().value(); + } + private String cloudAttr() { + if (zone == null) return null; + return zone.cloud().name().value(); + } + + private String getDeploymentCluster(ClusterSpec cluster) { + if (applicationId == null) return null; + if (zone == null) return null; + String appString = applicationId.toFullString(); + return String.join(".", appString, + zone.environment().value(), + zone.region().value(), + cluster.id().value()); + } + + private Map<String, String> serviceAttributes(Service svc) { + Map<String, String> dimvals = new LinkedHashMap<>(); + // currently unused - can be added if necessary: + // dimvals.put("local_service_name", svc.getServiceName()); + // dimvals.put("local_service_type", svc.getServiceType()); + String cName = svc.getServicePropertyString("clustername", null); + if (cName != null) { + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cName); + } + String cType = svc.getServicePropertyString("clustertype", null); + if (cType != null) { + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cType); + } + var hostResource = svc.getHost(); + if (hostResource != null) { + hostResource.spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { + dimvals.put(PublicDimensions.DEPLOYMENT_CLUSTER, getDeploymentCluster(cluster)); + // overrides value above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); + // alternative to above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); + cluster.group().ifPresent(group -> dimvals.put(PublicDimensions.GROUP_ID, group.toString())); + }); + } + return dimvals; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 40c9a03b126..02a6b243054 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -132,7 +132,7 @@ public class ConstantTensorJsonValidator { private void consumeTopObject() throws IOException { for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_TYPE -> consumeTypeField(); case FIELD_VALUES -> consumeValuesField(); @@ -189,7 +189,7 @@ public class ConstantTensorJsonValidator { } for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - validateNumeric(parser.getCurrentName(), parser.nextToken()); + validateNumeric(parser.currentName(), parser.nextToken()); } } @@ -199,7 +199,7 @@ public class ConstantTensorJsonValidator { boolean seenValue = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(tensorDimensions.keySet())); @@ -228,13 +228,13 @@ public class ConstantTensorJsonValidator { // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String dimensionName = parser.getCurrentName(); + String dimensionName = parser.currentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", dimensionName)); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", dimensionName)); } cellDimensions.remove(dimensionName); validateLabel(dimension); @@ -300,7 +300,7 @@ public class ConstantTensorJsonValidator { } private void assertCurrentTokenIs(JsonToken wantedToken) { - assertTokenIs(parser.getCurrentToken(), wantedToken); + assertTokenIs(parser.currentToken(), wantedToken); } private void assertNextTokenIs(JsonToken wantedToken) throws IOException { @@ -316,11 +316,11 @@ public class ConstantTensorJsonValidator { static class InvalidConstantTensorException extends IllegalArgumentException { InvalidConstantTensorException(JsonParser parser, String message) { - super(message + " " + parser.getCurrentLocation().toString()); + super(message + " " + parser.currentLocation().toString()); } InvalidConstantTensorException(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + super("Failed to parse JSON stream " + parser.currentLocation().toString(), base); } InvalidConstantTensorException(IOException base) { @@ -412,7 +412,7 @@ public class ConstantTensorJsonValidator { boolean seenValues = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(mappedDims)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 9cf5fe84c21..4900b56801c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -23,21 +23,22 @@ public class JvmHeapSizeValidator implements Validator { context.model().getContainerClusters().forEach((clusterId, appCluster) -> { var mp = appCluster.getMemoryPercentage().orElse(null); if (mp == null) return; - if (mp.availableMemoryGb().isEmpty()) { + if (mp.asAbsoluteGb().isEmpty()) { context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); return; } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + double availableMemoryGb = mp.asAbsoluteGb().getAsDouble(); + int percentageOfTotal = mp.ofContainerTotal().getAsInt(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", - mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); - if (mp.percentage() < percentLimit) { + percentageOfTotal, percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + if (percentageOfTotal < percentLimit) { context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", - clusterId, mp.percentage(), percentLimit, modelCostGb)); + clusterId, percentageOfTotal, percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { context.illegal( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 4d9386b5f19..ea579aaf5d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -2,12 +2,13 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation.Context; @@ -31,25 +32,35 @@ public class QuotaValidator implements Validator { @Override public void validate(Context context) { + var zone = context.deployState().zone(); + var exclusivity = new Exclusivity(zone, context.deployState().featureFlags().sharedHosts()); + var capacityPolicies = new CapacityPolicies(zone, exclusivity, context.model().applicationPackage().getApplicationId(), + context.deployState().featureFlags().adminClusterArchitecture()); var quota = context.deployState().getProperties().quota(); quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, context.model())); - quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context.model(), context.deployState().getProperties().zone())); + quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context, capacityPolicies)); } - private void validateBudget(BigDecimal budget, VespaModel model, Zone zone) { - var maxSpend = model.allClusters().stream() - .filter(id -> !adminClusterIds(model).contains(id)) - .map(id -> model.provisioned().all().getOrDefault(id, zeroCapacity)) - .mapToDouble(c -> c.maxResources().cost()) // TODO: This may be unspecified -> 0 - .sum(); + private void validateBudget(BigDecimal budget, Context context, + CapacityPolicies capacityPolicies) { + var zone = context.deployState().getProperties().zone(); + var application = context.model().applicationPackage().getApplicationId(); + + var maxSpend = 0.0; + for (var id : context.model().allClusters()) { + if (adminClusterIds(context.model()).contains(id)) continue; + var cluster = context.model().provisioned().clusters().get(id); + var capacity = context.model().provisioned().capacities().getOrDefault(id, zeroCapacity); + maxSpend += capacityPolicies.applyOn(capacity, cluster.isExclusive()).maxResources().cost(); + } - var actualSpend = model.allocatedHosts().getHosts().stream() + var actualSpend = context.model().allocatedHosts().getHosts().stream() .filter(hostSpec -> hostSpec.membership().get().cluster().type() != ClusterSpec.Type.admin) .mapToDouble(hostSpec -> hostSpec.advertisedResources().cost()) .sum(); if (Math.abs(actualSpend) < 0.01) { - log.warning("Deploying application " + model.applicationPackage().getApplicationId() + " with zero budget use. This is suspicious, but not blocked"); + log.warning("Deploying application " + application + " with zero budget use. This is suspicious, but not blocked"); return; } @@ -69,7 +80,7 @@ public class QuotaValidator implements Validator { /** Check that all clusters in the application do not exceed the quota max cluster size. */ private void validateMaxClusterSize(int maxClusterSize, VespaModel model) { - var invalidClusters = model.provisioned().all().entrySet().stream() + var invalidClusters = model.provisioned().capacities().entrySet().stream() .filter(entry -> entry.getValue() != null) .filter(entry -> { var cluster = entry.getValue(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index ed0804f7420..f073c0f27c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -19,9 +19,11 @@ import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeVal import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseValidator; import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; +import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForLocalLLMValidator; import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnnxModelChangesValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; import com.yahoo.yolean.Exceptions; @@ -129,6 +131,7 @@ public class Validation { new CertificateRemovalChangeValidator().validate(execution); new RedundancyValidator().validate(execution); new RestartOnDeployForOnnxModelChangesValidator().validate(execution); + new RestartOnDeployForLocalLLMValidator().validate(execution); } public interface Context { @@ -213,6 +216,9 @@ public class Validation { @Override public void require(ConfigChangeAction action) { + if (action instanceof VespaRestartAction && action.getServices().isEmpty()) + throw new IllegalStateException("restart actions must have services specified"); + actions.add(action); action.validationId().ifPresent(id -> invalid(id, action.getMessage())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index bfd100f40c9..3722dd84e29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Stream; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java index df8bf0e9b01..f86a3b31182 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -38,7 +39,8 @@ public class GlobalDocumentChangeValidator implements ChangeValidator { if ( ! context.deployState().validationOverrides().allows(ValidationId.globalDocumentChange, context.deployState().now())) context.invalid(ValidationId.globalDocumentChange, reason); else if (context.deployState().isHosted()) - context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason)); + context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason, + nextCluster.getSearch().getSearchNodes().stream().map(AbstractService::getServiceInfo).toList())); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java index 0d4776ad00a..ce24d11121c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -36,7 +36,7 @@ public class NodeResourceChangeValidator implements ChangeValidator { } private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { - return currentResources.memoryGb() != nextResources.memoryGb(); + return currentResources.memoryGiB() != nextResources.memoryGiB(); } private Optional<NodeResources> resourcesOf(ClusterSpec.Id clusterId, VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 5d7a8779005..a1f8a97d47f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -42,7 +42,7 @@ public class ResourcesReductionValidator implements ChangeValidator { NodeResources currentResources = current.totalResources(); NodeResources nextResources = next.totalResources(); if (nextResources.vcpu() < 0.5 * currentResources.vcpu() || - nextResources.memoryGb() < 0.5 * currentResources.memoryGb() || + nextResources.memoryGiB() < 0.5 * currentResources.memoryGiB() || nextResources.diskGb() < 0.5 * currentResources.diskGb()) context.invalid(ValidationId.resourcesReduction, "Resource reduction in '" + clusterId.value() + "' is too large: " + @@ -60,9 +60,9 @@ public class ResourcesReductionValidator implements ChangeValidator { * This will always yield specified node resources on hosted instances and never on self-hosted instances. */ private ClusterResources clusterResources(ClusterSpec.Id id, VespaModel model) { - if ( ! model.provisioned().all().containsKey(id)) return null; + if ( ! model.provisioned().capacities().containsKey(id)) return null; - ClusterResources resources = model.provisioned().all().get(id).maxResources(); + ClusterResources resources = model.provisioned().capacities().get(id).maxResources(); if ( ! resources.nodeResources().isUnspecified()) return resources; var containerCluster = model.getContainerClusters().get(id.value()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java new file mode 100644 index 00000000000..ba1f19ff68c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java @@ -0,0 +1,58 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.INFO; +import static java.util.stream.Collectors.toUnmodifiableSet; + +/** + * If using local LLMs, this validator will make sure that restartOnDeploy is set for + * configs for this cluster. + * + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidator implements ChangeValidator { + + public static final String LOCAL_LLM_COMPONENT = ai.vespa.llm.clients.LocalLLM.class.getName(); + + private static final Logger log = Logger.getLogger(RestartOnDeployForLocalLLMValidator.class.getName()); + + @Override + public void validate(ChangeContext context) { + var previousClustersWithLocalLLM = findClustersWithLocalLLMs(context.previousModel()); + var nextClustersWithLocalLLM = findClustersWithLocalLLMs(context.model()); + + // Only restart services if we use a local LLM in both the next and previous generation + for (var clusterId : intersect(previousClustersWithLocalLLM, nextClustersWithLocalLLM)) { + String message = "Need to restart services in %s due to use of local LLM".formatted(clusterId); + context.require(new VespaRestartAction(clusterId, message, + context.model().getContainerClusters().get(clusterId.value()).getContainers() + .stream().map(AbstractService::getServiceInfo).toList())); + log.log(INFO, message); + } + } + + private Set<ClusterSpec.Id> findClustersWithLocalLLMs(VespaModel model) { + return model.getContainerClusters().values().stream() + .filter(cluster -> cluster.getAllComponents().stream() + .anyMatch(component -> component.getClassId().getName().equals(LOCAL_LLM_COMPONENT))) + .map(ApplicationContainerCluster::id) + .collect(toUnmodifiableSet()); + } + + private Set<ClusterSpec.Id> intersect(Set<ClusterSpec.Id> a, Set<ClusterSpec.Id> b) { + Set<ClusterSpec.Id> result = new HashSet<>(a); + result.retainAll(b); + return result; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index 008a3fc5547..f95f0a6c079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -99,7 +100,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida log.log(INFO, message); cluster.onnxModelCostCalculator().setRestartOnDeploy(); cluster.onnxModelCostCalculator().store(); - actions.add(new VespaRestartAction(cluster.id(), message)); + actions.add(new VespaRestartAction(cluster.id(), message, cluster.getContainers().stream().map(AbstractService::getServiceInfo).toList())); } private static boolean enoughMemoryToAvoidRestart(ApplicationContainerCluster clusterInCurrentModel, @@ -111,11 +112,11 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida double currentModelCostInGb = onnxModelCostInGb(clusterInCurrentModel); double nextModelCostInGb = onnxModelCostInGb(cluster); - double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); + double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); double memoryUsedByModels = currentModelCostInGb + nextModelCostInGb; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - memoryUsedByModels); - var availableMemoryPercentage = cluster.availableMemoryPercentage(); + var availableMemoryPercentage = cluster.heapSizePercentageOfAvailable(); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); var prefix = "Validating Onnx models memory usage for %s".formatted(cluster); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index c0b55d856ab..e36fa5b7373 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -16,6 +16,7 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi private final boolean ignoreForInternalRedeploy; + /** <strong>This does <em>not</em> trigger restarts; you <em>need</em> the {@code ServiceInfo}!</strong>*/ public VespaRestartAction(ClusterSpec.Id id, String message) { this(id, message, List.of()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 692de1769d3..34df7e9d963 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -12,15 +12,18 @@ import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.Logserver; +import com.yahoo.vespa.model.admin.LogserverContainer; +import com.yahoo.vespa.model.admin.LogserverContainerCluster; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; -import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilderBase; import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -35,10 +38,13 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; - public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, + private final ConfigModelContext context; + + public DomAdminV2Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - super(applicationType, multitenant, configServerSpecs); + super(context.getApplicationType(), multitenant, configServerSpecs); + this.context = context; } private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { @@ -52,6 +58,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); var logserver = parseLogserver(deployState, admin, adminE); admin.setLogserver(logserver); + createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); if (deployState.featureFlags().logserverOtelCol()) { // for manual testing addOtelcol(admin, deployState, logserver.getHostResource()); @@ -65,6 +72,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { addLoggingSpecs(new ModelElement(adminE).child("logging"), admin); } + private void createContainerOnLogserverHost(DeployState deployState, Admin admin, HostResource hostResource) { + LogserverContainerCluster logServerCluster = new LogserverContainerCluster(admin, "logs", deployState); + ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); + logserverClusterModel.setCluster(logServerCluster); + + LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + container.useDynamicPorts(); + container.setHostResource(hostResource); + container.initService(deployState); + logServerCluster.addContainer(container); + admin.addAndInitializeService(deployState, hostResource, container); + admin.setLogserverContainerCluster(logServerCluster); + context.getConfigModelRepoAdder().add(logserverClusterModel); + } + private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers; if (multitenant) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 347bb504857..c6086327a1c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -111,6 +111,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { logserverClusterModel.setCluster(logServerCluster); LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + if (deployState.getProperties().applicationId().instance().isTester()) + container.useDynamicPorts(); // TODO: read current version in ApplicationRepository, and always use this. container.setHostResource(hostResource); container.initService(deployState); logServerCluster.addContainer(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index d877600db13..11f4c9794aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -137,7 +137,7 @@ public class NodesSpecification { int defaultMinGroups = nodes.from().orElse(1) / groupSize.to().orElse(nodes.from().orElse(1)); int defaultMaxGroups = groupSize.isEmpty() ? 1 : nodes.to().orElse(1) / groupSize.from().orElse(1); - var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); + var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); var max = new ClusterResources(nodes.to().orElse(1), groups.to().orElse(defaultMaxGroups), nodeResources(nodesElement).getSecond()); return new ResourceConstraints(min, max, groupSize); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index e3fc680ad6a..1f8d7fd7520 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -82,11 +82,6 @@ public final class ApplicationContainer extends Container implements builder.myid(index()); } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } private Optional<NodeResources> realResources() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index ed7646b3066..014d183071d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -209,22 +209,27 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { - int availableMemoryPercentage = availableMemoryPercentage(); - if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known - - // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); - double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); - double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); - int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); - return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); + int heapSizePercentageOfAvailable = heapSizePercentageOfAvailable(); + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(heapSizePercentageOfAvailable)); // Node memory is not known + + // Node memory is known, so compute heap size as a percentage of available memory (excluding overhead, which the startup scripts also account for) + double totalMemoryGb = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); + double totalMemoryMinusOverhead = Math.max(0, totalMemoryGb - Host.memoryOverheadGb); + double onnxModelCostGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); + double availableMemoryGb = Math.max(0, totalMemoryMinusOverhead - onnxModelCostGb); + int memoryPercentageOfAvailable = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryMinusOverhead); + int memoryPercentageOfTotal = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryGb); + logger.log(FINE, () -> ("cluster id '%s': memoryPercentageOfAvailable=%d, memoryPercentageOfTotal=%d, " + + "availableMemoryGb=%f, totalMemoryGb=%f, heapSizePercentageOfAvailable=%d, onnxModelCostGb=%f") + .formatted(id(), memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb, totalMemoryGb, heapSizePercentageOfAvailable, onnxModelCostGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb * heapSizePercentageOfAvailable * 1e-2)); } return Optional.empty(); } - public int availableMemoryPercentage() { + public int heapSizePercentageOfAvailable() { return getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; @@ -310,14 +315,23 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); var memoryPct = getMemoryPercentage().orElse(null); - int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() - ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; + int heapsize = truncateTo4SignificantBits(memoryPct != null && memoryPct.asAbsoluteGb().isPresent() + ? (int) (memoryPct.asAbsoluteGb().getAsDouble() * 1024) : 1536); builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) - .minHeapsize(heapsize) + .minHeapsize(heapsize) // These cause restarts when changed, so we try to keep them stable. .heapsize(heapsize); - if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.ofContainerAvailable()); + } + + static int truncateTo4SignificantBits(int i) { + if (i == Integer.MIN_VALUE) return i; + if (i < 0) return -truncateTo4SignificantBits(-i); + if (i <= 16) return i; + int mask = Integer.highestOneBit(i); + mask += mask - (mask >> 3); + return i & mask; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f1945ca5341..c2368fd29a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -105,15 +105,9 @@ public abstract class Container extends AbstractService implements addBuiltinHandlers(); addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); - - appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x1000000"); // 16M } - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - void setOwner(ContainerCluster<?> owner) { this.owner = owner; } /** True if this container is retired (slated for removal) */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index aeb6c030a49..00ab47e8ddb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -73,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeSet; @@ -721,10 +722,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { - static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } - static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { - return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + public record JvmMemoryPercentage(int ofContainerAvailable, OptionalInt ofContainerTotal, OptionalDouble asAbsoluteGb) { // optionalInt pctOfTotal < int pctOfAvailable + static JvmMemoryPercentage of(int percentageOfAvailable) { return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.empty(), OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentageOfAvailable, int percentageOfTotal, double absoluteMemoryGb) { + return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.of(percentageOfTotal), OptionalDouble.of(absoluteMemoryGb)); } } public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java index c210c2621a6..693eebd75a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java @@ -33,17 +33,15 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif if (spec != null) { for (Element modelElement : XML.getChildren(spec, "model")) { - addConfig( - modelElement.getAttribute("language"), - Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); + addConfig(Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); } } } - public void addConfig(String language, ModelReference path) { + public void addConfig(ModelReference path) { configList.add( - new SignificanceModelConfig(language, path) + new SignificanceModelConfig(path) ); } @@ -53,19 +51,16 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif builder.model( configList.stream() .map(config -> new SignificanceConfig.Model.Builder() - .language(config.language) .path(config.path) ).toList() ); } - class SignificanceModelConfig { - private final String language; + static class SignificanceModelConfig { private final ModelReference path; - public SignificanceModelConfig(String language, ModelReference path) { - this.language = language; + public SignificanceModelConfig(ModelReference path) { this.path = path; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 5f824950ecd..9c9e20062f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; +import ai.vespa.utils.BytesQuantity; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.security.tls.TlsContext; @@ -21,6 +22,8 @@ import java.util.TreeSet; */ public class HostedSslConnectorFactory extends ConnectorFactory { + private record EntityLoggingEntry(String prefix, double sampleRate, BytesQuantity maxEntitySize) {} + private final SslClientAuth clientAuth; private final List<String> tlsCiphersOverride; private final boolean proxyProtocolEnabled; @@ -28,6 +31,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; private final Set<String> knownServerNames; + private final List<EntityLoggingEntry> entityLoggingEntries; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -40,6 +44,22 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); this.knownServerNames = Collections.unmodifiableSet(new TreeSet<>(builder.knownServerNames)); + this.entityLoggingEntries = builder.requestPrefixForLoggingContent.stream() + .map(prefix -> { + var parts = prefix.split(":"); + if (parts.length != 3) { + throw new IllegalArgumentException("Expected string of format 'prefix:sample-rate:max-entity-size', got '%s'".formatted(prefix)); + } + var pathPrefix = parts[0]; + if (pathPrefix.isBlank()) + throw new IllegalArgumentException("Path prefix must not be blank"); + var sampleRate = Double.parseDouble(parts[1]); + if (sampleRate < 0 || sampleRate > 1) + throw new IllegalArgumentException("Sample rate must be in range [0, 1], got '%s'".formatted(sampleRate)); + var maxEntitySize = BytesQuantity.fromString(parts[2]); + return new EntityLoggingEntry(pathPrefix, sampleRate, maxEntitySize); + }) + .toList(); } private static SslProvider createSslProvider(Builder builder) { @@ -72,8 +92,14 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .idleTimeout(Duration.ofSeconds(30).toSeconds()) .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) .accessLog(new ConnectorConfig.AccessLog.Builder() - .remoteAddressHeaders(remoteAddressHeaders) - .remotePortHeaders(remotePortHeaders)) + .remoteAddressHeaders(remoteAddressHeaders) + .remotePortHeaders(remotePortHeaders) + .content(entityLoggingEntries.stream() + .map(e -> new ConnectorConfig.AccessLog.Content.Builder() + .pathPrefix(e.prefix) + .sampleRate(e.sampleRate) + .maxSize(e.maxEntitySize.toBytes())) + .toList())) .serverName.known(knownServerNames); } @@ -93,6 +119,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { String tlsCaCertificatesPath; boolean tokenEndpoint; Set<String> knownServerNames = Set.of(); + Set<String> requestPrefixForLoggingContent = Set.of(); private Builder(String name, int port) { this.name = name; this.port = port; } public Builder clientAuth(SslClientAuth auth) { clientAuth = auth; return this; } @@ -106,6 +133,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } public Builder knownServerNames(Set<String> knownServerNames) { this.knownServerNames = Set.copyOf(knownServerNames); return this; } + public Builder requestPrefixForLoggingContent(Collection<String> v) { this.requestPrefixForLoggingContent = Set.copyOf(v); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 56e2a21e38b..4995c20b985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -140,6 +140,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); + static final String HOSTED_VESPA_TENANT_PARENT_DOMAIN = "vespa.tenant."; + //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; @@ -235,6 +237,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); + addAthenzServiceIdentityProvider(cluster, context, deployState.getDeployLogger()); addParameterStoreValidationHandler(cluster, deployState); } @@ -344,6 +347,20 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(cloudSecretStore); } + private void addAthenzServiceIdentityProvider(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { + if ( ! context.getDeployState().isHosted()) return; + if ( ! context.getDeployState().zone().system().isPublic()) return; // Non-public is handled by deployment spec config. + if ( ! context.properties().launchApplicationAthenzService()) return; + addIdentityProvider(cluster, + context.getDeployState().getProperties().configServerSpecs(), + context.getDeployState().getProperties().loadBalancerName(), + context.getDeployState().getProperties().ztsUrl(), + context.getDeployState().getProperties().athenzDnsSuffix(), + context.getDeployState().zone(), + AthenzDomain.from(HOSTED_VESPA_TENANT_PARENT_DOMAIN + context.properties().applicationId().tenant().value()), + AthenzService.from(context.properties().applicationId().application().value())); + } + private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; DeploymentSpec deploymentSpec = app.getDeploymentSpec(); @@ -607,7 +624,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state)) .proxyProtocol(state.zone().cloud().useProxyProtocol()) .tlsCiphersOverride(state.getProperties().tlsCiphersOverride()) - .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()); + .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()); var endpointCert = state.endpointCertificateSecrets().orElse(null); if (endpointCert != null) { builder.endpointCertificate(endpointCert); @@ -670,6 +688,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .remotePortHeader("X-Forwarded-Port") .clientAuth(SslClientAuth.NEED) .knownServerNames(tokenEndpoints) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()) .build(); server.addConnector(connector); @@ -815,8 +834,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } - cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); } + for (OnnxModel onnxModel : models.asMap().values()) + cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); } @@ -1037,7 +1057,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign", e); + " must be given as an integer followe by '%'", e); } } @@ -1072,9 +1092,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return List.of(node); } + private static void requireFixedSizeSingularNodeIfTester(ConfigModelContext context, NodesSpecification nodes) { + if ( ! context.properties().hostedVespa() || ! context.properties().applicationId().instance().isTester()) + return; + + if ( ! nodes.maxResources().equals(nodes.minResources())) + throw new IllegalArgumentException("tester resources must be absolute, but min and max resources differ: " + nodes); + + if (nodes.maxResources().nodes() > 1) + throw new IllegalArgumentException("tester cannot run on more than 1 node, but " + nodes.maxResources().nodes() + " nodes were specified"); + } + private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + requireFixedSizeSingularNodeIfTester(context, nodesSpecification); var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, @@ -1280,37 +1312,36 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addIdentityProvider(ApplicationContainerCluster cluster, - List<ConfigServerSpec> configServerSpecs, - HostName loadBalancerName, - URI ztsUrl, - String athenzDnsSuffix, - Zone zone, - DeploymentSpec spec) { - spec.athenzDomain() - .ifPresent(domain -> { - AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) - .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + - app.getApplicationId().instance() + "'")); - String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; - IdentityProvider identityProvider = new IdentityProvider(domain, - service, - getLoadBalancerName(loadBalancerName, configServerSpecs), - ztsUrl, - zoneDnsSuffix, - zone); - - // Replace AthenzIdentityProviderProvider - cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); - cluster.addComponent(identityProvider); - - var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; - cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); - - cluster.getContainers().forEach(container -> { - container.setProp("identity.domain", domain.value()); - container.setProp("identity.service", service.value()); - }); + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, DeploymentSpec spec) { + spec.athenzDomain().ifPresent(domain -> { + AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) + .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + + app.getApplicationId().instance() + "'")); + addIdentityProvider(cluster, configServerSpecs, loadBalancerName, ztsUrl, athenzDnsSuffix, zone, domain, service); + }); + } + + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, AthenzDomain domain, AthenzService service) { + String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; + IdentityProvider identityProvider = new IdentityProvider(domain, + service, + getLoadBalancerName(loadBalancerName, configServerSpecs), + ztsUrl, + zoneDnsSuffix, + zone); + + // Replace AthenzIdentityProviderProvider + cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); + cluster.addComponent(identityProvider); + + var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; + cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); + + cluster.getContainers().forEach(container -> { + container.setProp("identity.domain", domain.value()); + container.setProp("identity.service", service.value()); }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fbac9e9d710..994837a2dbe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -237,24 +236,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; - TransactionLogServer tls; if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); - - tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); - tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + searchNode = new SearchNode.Builder("" + node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved, syncTransactionLog) .build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } - searchNode.setTls(tls); if (searchCluster != null) { searchCluster.addSearcher(searchNode); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 15f3eece8a9..c39da176828 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -22,11 +22,6 @@ public class DispatchTuning { private final Integer maxHitsPerPartition; private final DispatchPolicy dispatchPolicy; private final Double minActiveDocsCoverage; - - public Double getTopkProbability() { - return topkProbability; - } - private final Double topkProbability; private DispatchTuning(Builder builder) { @@ -45,6 +40,8 @@ public class DispatchTuning { /** Returns the percentage of documents which must be available in a group for that group to receive queries */ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } + public Double getTopkProbability() { return topkProbability; } + public static class Builder { private Integer maxHitsPerPartition; @@ -71,14 +68,14 @@ public class DispatchTuning { } public static DispatchPolicy toDispatchPolicy(String policy) { - switch (policy.toLowerCase()) { - case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 - case "round-robin": return DispatchPolicy.ROUNDROBIN; - case "latency-amortized-over-requests" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; - case "latency-amortized-over-time" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; - case "best-of-random-2" : return DispatchPolicy.BEST_OF_RANDOM_2; - default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); - } + return switch (policy.toLowerCase()) { + case "adaptive", "random" -> DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 + case "round-robin" -> DispatchPolicy.ROUNDROBIN; + case "latency-amortized-over-requests" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; + case "latency-amortized-over-time" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; + case "best-of-random-2" -> DispatchPolicy.BEST_OF_RANDOM_2; + default -> throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); + }; } public Builder setMinActiveDocsCoverage(Double minCoverage) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index c182f0e0507..f218cedb74e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -35,6 +35,9 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; private final int contentLayerMetadataFeatureLevel; + private final boolean symmetricPutAndActivateReplicaSelection; + private final boolean enforceStrictlyIncreasingClusterStateVersions; + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -97,19 +100,25 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen var featureFlags = deployState.getProperties().featureFlags(); int maxInhibitedGroups = featureFlags.maxActivationInhibitedOutOfSyncGroups(); int contentLayerMetadataFeatureLevel = featureFlags.contentLayerMetadataFeatureLevel(); + boolean symmetricPutAndActivateReplicaSelection = featureFlags.symmetricPutAndActivateReplicaSelection(); + boolean enforceStrictlyIncreasingClusterStateVersions = featureFlags.enforceStrictlyIncreasingClusterStateVersions(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, maxInhibitedGroups, - contentLayerMetadataFeatureLevel); + contentLayerMetadataFeatureLevel, + symmetricPutAndActivateReplicaSelection, + enforceStrictlyIncreasingClusterStateVersions); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, int maxActivationInhibitedOutOfSyncGroups, - int contentLayerMetadataFeatureLevel) + int contentLayerMetadataFeatureLevel, + boolean symmetricPutAndActivateReplicaSelection, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "distributor"); this.parent = parent; @@ -118,6 +127,8 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.contentLayerMetadataFeatureLevel = contentLayerMetadataFeatureLevel; + this.symmetricPutAndActivateReplicaSelection = symmetricPutAndActivateReplicaSelection; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -132,6 +143,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen if (contentLayerMetadataFeatureLevel > 0) { builder.enable_operation_cancellation(true); } + builder.symmetric_put_and_activate_replica_selection(symmetricPutAndActivateReplicaSelection); bucketSplitting.getConfig(builder); } @@ -152,6 +164,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen builder.root_folder(""); builder.cluster_name(parent.getName()); builder.is_distributor(true); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } public String getClusterName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bac86e37e8f..d37e5d5382a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -46,6 +46,7 @@ import com.yahoo.vespa.model.content.IndexedHierarchicDistributionValidator; import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ReservedDocumentTypeNameValidator; import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; @@ -137,6 +138,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming()); c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits); validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); + warnIfDistributionKeyRangeIsSuboptimal(c.clusterId, c.rootGroup, deployState); c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); @@ -247,7 +249,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().ofContainerAvailable() * 0.01; } } return 0.0; @@ -275,6 +277,38 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem } } + private static class HighestDistributionKeyAggregator { + public int nodeCount = 0; + public int highestNodeDistributionKey = 0; + + void aggregateNodeStats(StorageGroup group) { + for (StorageNode n : group.getNodes()) { + nodeCount++; + highestNodeDistributionKey = Math.max(highestNodeDistributionKey, n.getDistributionKey()); + } + for (StorageGroup g : group.getSubgroups()) { + aggregateNodeStats(g); + } + } + } + + private void warnIfDistributionKeyRangeIsSuboptimal(String clusterId, StorageGroup rootGroup, DeployState deployState) { + if (rootGroup == null) { + return; // Unit testing case + } + var aggr = new HighestDistributionKeyAggregator(); + aggr.aggregateNodeStats(rootGroup); + int warnThreshold = 100; // ... Not scientifically chosen + if ((aggr.highestNodeDistributionKey - aggr.nodeCount) >= warnThreshold) { + deployState.getDeployLogger().logApplicationPackage(WARNING, + ("Content cluster '%s' has %d node(s), but the highest distribution key is %d. " + + "Having much higher distribution keys than the number of nodes is not recommended, " + + "as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node") + .formatted(clusterId, aggr.nodeCount, aggr.highestNodeDistributionKey)); + } + } + private void addClusterControllers(ConfigModelContext context, ModelElement contentElement, ContentCluster contentCluster, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 701da93a329..8bde038b160 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -31,13 +31,15 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> protected StorageCluster doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { final ModelElement clusterElem = new ModelElement(producerSpec); final ContentCluster cluster = (ContentCluster)ancestor; + var featureFlags = deployState.getProperties().featureFlags(); return new StorageCluster(ancestor, ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem), new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem), new StorVisitorProducer.Builder().build(clusterElem), - new PersistenceProducer.Builder().build(clusterElem)); + new PersistenceProducer.Builder().build(clusterElem), + featureFlags.enforceStrictlyIncreasingClusterStateVersions()); } } @@ -46,19 +48,22 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> private final StorServerProducer storServerProducer; private final StorVisitorProducer storVisitorProducer; private final PersistenceProducer persistenceProducer; + private final boolean enforceStrictlyIncreasingClusterStateVersions; StorageCluster(TreeConfigProducer<?> parent, String clusterName, FileStorProducer fileStorProducer, StorServerProducer storServerProducer, StorVisitorProducer storVisitorProducer, - PersistenceProducer persistenceProducer) { + PersistenceProducer persistenceProducer, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "storage"); this.clusterName = clusterName; this.fileStorProducer = fileStorProducer; this.storServerProducer = storServerProducer; this.storVisitorProducer = storVisitorProducer; this.persistenceProducer = persistenceProducer; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -85,6 +90,7 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> @Override public void getConfig(StorServerConfig.Builder builder) { storServerProducer.getConfig(builder); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 2616dd8a93c..d60e6ac2df6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -21,8 +20,9 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 11.0; private final static double TLS_SIZE_FRACTION = 0.02; - final static long MB = 1024 * 1024; - public final static long GB = MB * 1024; + final static long MiB = 1024 * 1024; + public final static long GiB = MiB * 1024; + public final static long GB = 1_000_000_000; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; @@ -50,14 +50,14 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GiB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { builder.hwinfo.disk.shared(true); builder.hwinfo.cpu.cores((int)resources.vcpu()); - builder.hwinfo.memory.size((long)(usableMemoryGb() * GB)); + builder.hwinfo.memory.size((long)(usableMemoryGb() * GiB)); builder.hwinfo.disk.size((long)(resources.diskGb() * GB)); } @@ -68,12 +68,12 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - long fileSizeBytes = (long) Math.max(256*MB, usableMemoryGb()*GB*SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); + long fileSizeBytes = (long) Math.max(256* MiB, usableMemoryGb()* GiB *SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); builder.maxfilesize(fileSizeBytes); } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GiB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } @@ -89,7 +89,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB); - tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); + tlsSizeBytes = max(2* GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } @@ -114,7 +114,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { - double usableMemoryGb = resources.memoryGb() - Host.memoryOverheadGb; + double usableMemoryGb = resources.memoryGiB() - Host.memoryOverheadGb; return usableMemoryGb * (1 - fractionOfMemoryReserved); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 61933c10504..08743290ae3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -55,17 +55,18 @@ public class SearchNode extends AbstractService implements private static final int UNUSED_2 = 2; private static final int UNUSED_3 = 3; private static final int HEALTH_PORT = 4; + private static final int TLS_PORT = 5; private final boolean isHostedVespa; private final boolean flushOnShutdown; private final NodeSpec nodeSpec; private final int distributionKey; private final String clusterName; - private TransactionLogServer tls; private final AbstractService serviceLayerService; private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -77,10 +78,11 @@ public class SearchNode extends AbstractService implements private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -89,6 +91,7 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.syncTransactionLog = syncTransactionLog; } @Override @@ -96,7 +99,7 @@ public class SearchNode extends AbstractService implements Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); } } @@ -105,9 +108,9 @@ public class SearchNode extends AbstractService implements String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved, - ModelContext.FeatureFlags featureFlags) { + ModelContext.FeatureFlags featureFlags, Boolean syncTransactionLog) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, syncTransactionLog); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -120,7 +123,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -134,9 +137,11 @@ public class SearchNode extends AbstractService implements portsMeta.on(UNUSED_2).tag("unused"); portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); + portsMeta.on(TLS_PORT).tag("tls"); // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.syncTransactionLog = syncTransactionLog; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -172,6 +177,7 @@ public class SearchNode extends AbstractService implements from.allocatePort("unused/2"); from.allocatePort("unused/3"); from.allocatePort("health"); + from.allocatePort("tls"); } /** @@ -181,7 +187,7 @@ public class SearchNode extends AbstractService implements */ @Override public int getPortCount() { - return 5; + return 6; } /** @@ -198,6 +204,8 @@ public class SearchNode extends AbstractService implements return getHttpPort(); } + int getTlsPort() { return getRelativePort(TLS_PORT); } + @Override public String getServiceType() { return "searchnode"; @@ -219,7 +227,10 @@ public class SearchNode extends AbstractService implements builder.usefsync(false); } } - tls.getConfig(builder); + builder.listenport(getTlsPort()) + .basedir(getTlsDir()); + if (syncTransactionLog != null) + builder.usefsync(syncTransactionLog); } @Override @@ -227,14 +238,6 @@ public class SearchNode extends AbstractService implements return getHostName(); } - private TransactionLogServer getTransactionLogServer() { - return tls; - } - - public void setTls(TransactionLogServer tls) { - this.tls = tls; - } - public AbstractService getServiceLayerService() { return serviceLayerService; } @@ -260,7 +263,7 @@ public class SearchNode extends AbstractService implements httpport(getHttpPort()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostname() + ":" + getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). @@ -305,6 +308,8 @@ public class SearchNode extends AbstractService implements new MetricsmanagerConfig.Consumer.Builder().name("log").tags("logdefault")); } + private String getTlsDir() { return "tls";} + @Override public Optional<String> getPreShutdownCommand() { if (flushOnShutdown) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java deleted file mode 100644 index 5617fd15cbc..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchlib.TranslogserverConfig; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import org.w3c.dom.Element; - -/** - * @author hmusum - */ -public class TransactionLogServer extends AbstractService { - - private final Boolean useFsync; - - public TransactionLogServer(TreeConfigProducer<?> searchNode, String clusterName, Boolean useFsync) { - super(searchNode, "transactionlogserver"); - portsMeta.on(0).tag("tls"); - this.useFsync = useFsync; - setProp("clustername", clusterName); - setProp("clustertype", "search"); - } - - public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<TransactionLogServer> { - - private final String clusterName; - private final Boolean useFsync; - public Builder(String clusterName, Boolean useFsync) { - this.clusterName = clusterName; - this.useFsync = useFsync; - } - - @Override - protected TransactionLogServer doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName, useFsync); - } - - } - - public int getPortCount() { - return 1; - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { - // NB: ignore "start" - from.allocatePort("tls"); - } - - /** - * Returns the port used by the TLS. - * - * @return The port. - */ - int getTlsPort() { - return getRelativePort(0); - } - - /** - * Returns the directory used by the TLS. - * - * @return The directory. - */ - private String getTlsDir() { - return "tls"; - } - - public void getConfig(TranslogserverConfig.Builder builder) { - builder.listenport(getTlsPort()) - .basedir(getTlsDir()); - if (useFsync != null) { - builder.usefsync(useFsync); - } - } - -} diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 255cc3cde70..c9eff88764f 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -183,11 +183,14 @@ TOKEN : | < GRAM_SIZE: "gram-size" > | < MAX_LENGTH: "max-length" > | < MAX_OCCURRENCES: "max-occurrences" > +| < MAX_TOKEN_LENGTH: "max-token-length" > | < PREFIX: "prefix" > | < SUBSTRING: "substring" > | < SUFFIX: "suffix" > | < CONSTANT: "constant"> | < ONNX_MODEL: "onnx-model"> +| < SIGNIFICANCE: "significance"> +| < USE_MODEL: "use-model"> | < INTRAOP_THREADS: "intraop-threads"> | < INTEROP_THREADS: "interop-threads"> | < GPU_DEVICE: "gpu-device"> @@ -1366,7 +1369,8 @@ void matchType(ParsedMatchSettings matchInfo) : { } */ void matchItem(ParsedMatchSettings matchInfo) : { } { - ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | maxTermOccurrences(matchInfo)) + ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | + maxTermOccurrences(matchInfo) | maxTokenLength(matchInfo) ) } void exactTerminator(ParsedMatchSettings matchInfo) : @@ -1411,6 +1415,16 @@ void maxTermOccurrences(ParsedMatchSettings matchInfo) : } } +void maxTokenLength(ParsedMatchSettings matchInfo) : +{ + int maxTokenLength; +} +{ + <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() { + matchInfo.setMaxTokenLength(maxTokenLength); + } +} + /** * Consumes a rank statement of a field element. * @@ -1742,6 +1756,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) + | diversity(profile) | function(profile) | mutate(profile) | ignoreRankFeatures(profile) @@ -1761,7 +1776,8 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | matchFeatures(profile) | summaryFeatures(profile) | onnxModelInProfile(profile) - | strict(profile) ) + | strict(profile) + | significance(profile)) } /** @@ -1860,14 +1876,14 @@ void matchPhase(ParsedRankProfile profile) : MatchPhaseSettings settings = new MatchPhaseSettings(); } { - <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> + <MATCH_PHASE> lbrace() (matchPhaseItem(profile, settings) (<NL>)*)* <RBRACE> { settings.checkValid(); - profile.setMatchPhaseSettings(settings); + profile.setMatchPhase(settings); } } -void matchPhaseItem(MatchPhaseSettings settings) : +void matchPhaseItem(ParsedRankProfile profile, MatchPhaseSettings settings) : { String str; int num; @@ -1876,7 +1892,7 @@ void matchPhaseItem(MatchPhaseSettings settings) : } { ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } - | diversity(settings) + | diversityDeprecated(profile) | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); } @@ -1891,7 +1907,18 @@ void matchPhaseItem(MatchPhaseSettings settings) : * * @param profile The rank profile to modify. */ -void diversity(MatchPhaseSettings profile) : +void diversity(ParsedRankProfile profile) : +{ + DiversitySettings settings = new DiversitySettings(); +} +{ + <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE> + { + profile.setDiversity(settings); + } +} + +void diversityDeprecated(ParsedRankProfile profile) : { DiversitySettings settings = new DiversitySettings(); } @@ -1899,6 +1926,7 @@ void diversity(MatchPhaseSettings profile) : <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE> { profile.setDiversity(settings); + deployLogger.logApplicationPackage(Level.WARNING, "'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level."); } } @@ -1966,10 +1994,12 @@ void secondPhaseItem(ParsedRankProfile profile) : { String expression; int rerankCount; + double dropLimit; } { ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } + | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setSecondPhaseRankScoreDropLimit(dropLimit); } ) } @@ -2115,6 +2145,22 @@ void strict(ParsedRankProfile profile) : ) } +void significance(ParsedRankProfile profile) : +{} +{ + <SIGNIFICANCE> lbrace() (significanceItem(profile) (<NL>)*)* <RBRACE> + {} +} + +void significanceItem(ParsedRankProfile profile) : +{} +{ + <USE_MODEL> <COLON> ( + ( <TRUE> { profile.setUseSignificanceModel(true); } ) | + ( <FALSE> { profile.setUseSignificanceModel(false); } ) + ) +} + /** * Consumes a match-features block of a rank profile. * @@ -2710,6 +2756,7 @@ String identifierWithDash() : | <TARGET_HITS_MAX_ADJUSTMENT_FACTOR> | <TERMWISE_LIMIT> | <UPPER_BOUND> + | <USE_MODEL> ) { return token.image; } } @@ -2812,6 +2859,7 @@ String identifier() : { } | <STEMMING> | <STRENGTH> | <STRICT> + | <SIGNIFICANCE> | <STRING> | <STRUCT> | <SUBSTRING> diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 08092f10020..c79a7b38d09 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -138,7 +138,7 @@ Threadpool = element threadpool { } Significance = element significance { - element model { attribute language { xsd:string } & ModelReference }* + element model { ModelReference }* } Clients = element clients { diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 3491d868f20..f79fc614a53 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -2,6 +2,8 @@ # RELAX NG Compact Syntax # Vespa Deployment file +include "common.rnc" + start = element deployment { attribute version { "1.0" } & attribute major-version { text }? & @@ -100,7 +102,7 @@ Test = element test { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Staging = element staging { @@ -108,7 +110,7 @@ Staging = element staging { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Dev = element dev { @@ -129,7 +131,8 @@ Prod = element prod { Region* & Delay* & ProdTest* & - ParallelSteps* + ParallelSteps* & + Tester? } ProdTest = element test { @@ -197,3 +200,7 @@ MemberRegion = element region { attribute fraction { xsd:double }? & text } + +Tester = element tester { + Nodes? +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_serving/services.xml b/config-model/src/test/cfg/application/ml_serving/services.xml index 3a5a4438c78..b1271b1297f 100644 --- a/config-model/src/test/cfg/application/ml_serving/services.xml +++ b/config-model/src/test/cfg/application/ml_serving/services.xml @@ -3,7 +3,13 @@ <services version="1.0"> <container version="1.0"> - <model-evaluation/> + <model-evaluation> + <onnx> + <models> + <model name="sqrt" /> <!-- list one of the models --> + </models> + </onnx> + </model-evaluation> <nodes> <node hostalias="node1" /> </nodes> diff --git a/config-model/src/test/cfg/significance/services.xml b/config-model/src/test/cfg/significance/services.xml index 6991f5498fb..ffdb73bfc2e 100644 --- a/config-model/src/test/cfg/significance/services.xml +++ b/config-model/src/test/cfg/significance/services.xml @@ -8,9 +8,9 @@ <container version="1.0"> <search> <significance> - <model language="en" model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> - <model language="no" path="models/idf-norwegian-wiki.json.zst" /> - <model language="ru" url="https://some/uri/blob.json" /> + <model model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> + <model path="models/idf-norwegian-wiki.json.zst" /> + <model url="https://some/uri/blob.json" /> </significance> </search> </container> diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg index 51a49502b64..d633cd97f0c 100644 --- a/config-model/src/test/derived/advanced/ilscripts.cfg +++ b/config-model/src/test/derived/advanced/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "advanced" ilscript[].docfield[] "debug_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg index 767c3af3c19..53dc789fbb7 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg index d8e6c882b80..b0a69c5408a 100644 --- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance" diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg index ae4ea621583..5ec1f839429 100644 --- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance2" diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg index 812f5e44545..eaa20043be8 100644 --- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsreference" diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg index 9d0962df5be..af179221eb4 100644 --- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg +++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationssimple" diff --git a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg index 0927045528d..a4fbe6f6c39 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "fast-search" @@ -16,6 +18,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg index 98cff642d9e..3f2dae48552 100644 --- a/config-model/src/test/derived/arrays/ilscripts.cfg +++ b/config-model/src/test/derived/arrays/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "arrays" ilscript[].docfield[] "tags" diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg index dec054b33f0..5a3784f7cb9 100644 --- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg +++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefetch" ilscript[].docfield[] "singlebyte" diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg index 6d3ef2799d9..58279759e5f 100644 --- a/config-model/src/test/derived/attributes/ilscripts.cfg +++ b/config-model/src/test/derived/attributes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "attributes" ilscript[].docfield[] "a1" diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg index 1d4e8f485b3..245cff48d15 100644 --- a/config-model/src/test/derived/attributes/index-info.cfg +++ b/config-model/src/test/derived/attributes/index-info.cfg @@ -175,8 +175,6 @@ indexinfo[].command[].indexname "a13" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "a13" indexinfo[].command[].command "type tensor(x{})" -indexinfo[].command[].indexname "a13" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "a7_arr" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "a7_arr" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg index c20c321ebcf..0b925da4778 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg +++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "str_1" diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg index 4405d2fda40..7d025e15703 100644 --- a/config-model/src/test/derived/complex/ilscripts.cfg +++ b/config-model/src/test/derived/complex/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "complex" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg index e4242153bce..bbb8e5c556c 100644 --- a/config-model/src/test/derived/emptydefault/ilscripts.cfg +++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "emptydefault" ilscript[].docfield[] "one" diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 21dfbd1371b..1d1bd6d5e8a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index 4a9ba85ad38..73a80d3510e 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -22,6 +22,8 @@ indexinfo[].command[].command "exact *!A!!*" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "string" @@ -30,6 +32,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "exact *!B!!*" indexinfo[].command[].indexname "string_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "type string" @@ -38,6 +42,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "string_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "type string" @@ -58,6 +64,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "elem_map" indexinfo[].command[].command "multivalue" @@ -66,6 +74,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "string" @@ -74,6 +84,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "exact @@" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "integer" @@ -84,6 +96,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "elem_array" indexinfo[].command[].command "type Array<elem>" indexinfo[].command[].indexname "another_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "type string" @@ -104,6 +118,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "another_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "another_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "another_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg index e48f116f468..c811b93c3df 100644 --- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "t1" diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg index d3ab29f6cd8..121e305059e 100644 --- a/config-model/src/test/derived/id/ilscripts.cfg +++ b/config-model/src/test/derived/id/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "id" ilscript[].docfield[] "uri" diff --git a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg index f820ad9720b..5a474f62e07 100644 --- a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg @@ -53,6 +53,8 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index 2b8a6fc344d..7c808c932b2 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -12,6 +12,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "fast-search" @@ -22,6 +24,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "numerical" @@ -54,10 +58,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "my_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "my_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "fast-search" @@ -74,6 +82,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 8c2349e37ea..e764fea8d1f 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -260,6 +260,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "f10.text" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "normalize" @@ -270,6 +272,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "type string" diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 472c1f95cb0..454f675c0a2 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg index d4c804773f0..c966f32a502 100644 --- a/config-model/src/test/derived/inheritance/ilscripts.cfg +++ b/config-model/src/test/derived/inheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "onlygrandparent" diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg index 1860f180839..d0abc08f1e0 100644 --- a/config-model/src/test/derived/language/ilscripts.cfg +++ b/config-model/src/test/derived/language/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "language" ilscript[].docfield[] "language" diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg index 8ba4bfa3349..49515e50df4 100644 --- a/config-model/src/test/derived/lowercase/ilscripts.cfg +++ b/config-model/src/test/derived/lowercase/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "lowercase" ilscript[].docfield[] "single_field_source" diff --git a/config-model/src/test/derived/map_attribute/index-info.cfg b/config-model/src/test/derived/map_attribute/index-info.cfg index 64a51db1edd..be9612ba293 100644 --- a/config-model/src/test/derived/map_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "fast-search" @@ -18,6 +20,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "string" @@ -30,6 +34,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "numerical" @@ -38,6 +44,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg index b649fa2aa8a..5f414f14420 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "fast-search" @@ -34,12 +36,16 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "str_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "str_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "numerical" @@ -66,6 +72,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "int_elem_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg index e91784db387..f96bc5ad99f 100644 --- a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg @@ -4,6 +4,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -24,6 +26,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg index 1304354a722..22214e83698 100644 --- a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -52,6 +58,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -60,6 +68,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -88,6 +98,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg index 75273abefa1..c0d16de6393 100644 --- a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "string" @@ -50,6 +54,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "string" @@ -60,6 +66,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "string" @@ -88,6 +96,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "string" @@ -98,6 +108,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "string" @@ -124,6 +136,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "string" @@ -134,6 +148,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg index 2ef04a85a4f..51c13848a05 100644 --- a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "type string" @@ -14,6 +16,8 @@ indexinfo[].command[].command "exact @mss3_key@" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "type string" @@ -28,6 +32,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -48,6 +54,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -56,6 +64,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg index cf6a2fc5992..c4cf9cd2f1e 100644 --- a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -54,6 +60,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -62,6 +70,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -90,6 +100,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 0cdf921de25..4a6de4154f8 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "multiplesummaries" ilscript[].docfield[] "a" diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg index f90cdb15baa..f79e8824b69 100644 --- a/config-model/src/test/derived/music/ilscripts.cfg +++ b/config-model/src/test/derived/music/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "music" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg index ab9a96f819b..ec06d01f05a 100644 --- a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg +++ b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg @@ -3,25 +3,25 @@ searchall 1 fieldspec[].name "vec_a" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "EUCLIDEAN" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_b" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "ANGULAR" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_c" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "INNERPRODUCT" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_d" fieldspec[].searchmethod NONE fieldspec[].arg1 "" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE documenttype[].name "test" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index 728856abbf2..8f59c21e97f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -156,6 +156,7 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -173,9 +174,11 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "defaultRankProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -193,6 +196,7 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "neuralNetworkProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg index b02e09a0496..487d2fca902 100644 --- a/config-model/src/test/derived/newrank/ilscripts.cfg +++ b/config-model/src/test/derived/newrank/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "newrank" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg index 0ed1589af0a..4918e23efc6 100644 --- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg +++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "orderilscripts" ilscript[].docfield[] "foo" diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg index ecafbc4a025..3f7611b25d8 100644 --- a/config-model/src/test/derived/position_array/ilscripts.cfg +++ b/config-model/src/test/derived/position_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_array" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_array/index-info.cfg b/config-model/src/test/derived/position_array/index-info.cfg index 5484964149e..65587820bc1 100644 --- a/config-model/src/test/derived/position_array/index-info.cfg +++ b/config-model/src/test/derived/position_array/index-info.cfg @@ -4,12 +4,16 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "pos.x" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "pos.y" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg index d2fc8503ce5..fbd1a293418 100644 --- a/config-model/src/test/derived/position_attribute/ilscripts.cfg +++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_attribute" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg index a86dcec92ec..4645798723c 100644 --- a/config-model/src/test/derived/position_extra/ilscripts.cfg +++ b/config-model/src/test/derived/position_extra/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_extra" ilscript[].docfield[] "pos_str" diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg index 40c7843a0a4..2d1904cf9d8 100644 --- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg +++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefixexactattribute" ilscript[].docfield[] "indexfield0" diff --git a/config-model/src/test/derived/rankingexpression/schema-info.cfg b/config-model/src/test/derived/rankingexpression/schema-info.cfg index 5bf01f10ede..f78eb7de310 100644 --- a/config-model/src/test/derived/rankingexpression/schema-info.cfg +++ b/config-model/src/test/derived/rankingexpression/schema-info.cfg @@ -148,96 +148,125 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures true +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "static" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "overflow" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "duplicates" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited-two-levels" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withmf" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withboolean" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withglobalphase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "layered" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withnorm" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withfusion" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-as-summary-feature" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-in-global-phase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withstringcompare" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(myquerystring)" schema[].rankprofile[].input[].type "string" schema[].rankprofile[].input[].name "query(mybadlong)" diff --git a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg index 377c10d3293..68892737e63 100644 --- a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg +++ b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg @@ -18,24 +18,32 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema0" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema3" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile new file mode 100644 index 00000000000..40e77c8c6be --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile @@ -0,0 +1,8 @@ +rank-profile outside_schema1 { + + // Comment with wrong comment character + function foo() { + expression: now + } + +} diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg index adcd2f70c70..22526d1aa23 100644 --- a/config-model/src/test/derived/ranktypes/ilscripts.cfg +++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "ranktypes" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/schemainheritance/ilscripts.cfg b/config-model/src/test/derived/schemainheritance/ilscripts.cfg index f7324920fe7..b1ba947f1dc 100644 --- a/config-model/src/test/derived/schemainheritance/ilscripts.cfg +++ b/config-model/src/test/derived/schemainheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "pf1" diff --git a/config-model/src/test/derived/schemainheritance/schema-info.cfg b/config-model/src/test/derived/schemainheritance/schema-info.cfg index 9fe71780c7a..466e66ad0bb 100644 --- a/config-model/src/test/derived/schemainheritance/schema-info.cfg +++ b/config-model/src/test/derived/schemainheritance/schema-info.cfg @@ -116,12 +116,16 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "child_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "parent_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/structandfieldset/index-info.cfg b/config-model/src/test/derived/structandfieldset/index-info.cfg index 87a95f5a908..0a423530cfa 100644 --- a/config-model/src/test/derived/structandfieldset/index-info.cfg +++ b/config-model/src/test/derived/structandfieldset/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.first_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.last_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg index c07f04b3021..a806bc1b712 100644 --- a/config-model/src/test/derived/structanyorder/ilscripts.cfg +++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" ilscript[].docfield[] "structfield" diff --git a/config-model/src/test/derived/structanyorder/index-info.cfg b/config-model/src/test/derived/structanyorder/index-info.cfg index 6ea63818572..43dc6312d3e 100644 --- a/config-model/src/test/derived/structanyorder/index-info.cfg +++ b/config-model/src/test/derived/structanyorder/index-info.cfg @@ -180,10 +180,14 @@ indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -344,6 +348,8 @@ indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s3.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield.s3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s3" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "numerical" @@ -352,6 +358,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/tensor/index-info.cfg b/config-model/src/test/derived/tensor/index-info.cfg index c9ce2433e17..2402f074837 100644 --- a/config-model/src/test/derived/tensor/index-info.cfg +++ b/config-model/src/test/derived/tensor/index-info.cfg @@ -9,26 +9,18 @@ indexinfo[].command[].indexname "f2" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f2" indexinfo[].command[].command "type tensor<float>(x[2],y[1])" -indexinfo[].command[].indexname "f2" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f3" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f3" indexinfo[].command[].command "type tensor(x{})" -indexinfo[].command[].indexname "f3" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f4" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f4" indexinfo[].command[].command "type tensor(x[10],y[10])" -indexinfo[].command[].indexname "f4" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f5" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f5" indexinfo[].command[].command "type tensor<float>(x[10])" -indexinfo[].command[].indexname "f5" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f6" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f6" @@ -39,5 +31,3 @@ indexinfo[].command[].indexname "f7" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f7" indexinfo[].command[].command "type tensor<int8>(p{},x[5])" -indexinfo[].command[].indexname "f7" -indexinfo[].command[].command "word" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg index c08b6a54c83..cad8ec81879 100644 --- a/config-model/src/test/derived/tokenization/ilscripts.cfg +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "tokenization" ilscript[].docfield[] "text" diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 17bed90deb4..73befb221ce 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "types" ilscript[].docfield[] "abyte" diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index cc49e006f98..6b39e4d1924 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,10 +96,14 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -110,6 +114,8 @@ indexinfo[].command[].command "type Array<sct>" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "normalize" @@ -122,6 +128,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "normalize" @@ -140,10 +148,14 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "stringmapfield" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "intmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "intmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "integer" @@ -154,10 +166,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "intmapfield" indexinfo[].command[].command "type Map<string,int>" indexinfo[].command[].indexname "floatmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "floatmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "type float" @@ -166,12 +182,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "floatmapfield" indexinfo[].command[].command "type Map<string,float>" indexinfo[].command[].indexname "longmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "longmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "integer" @@ -182,12 +202,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "longmapfield" indexinfo[].command[].command "type Map<int,long>" indexinfo[].command[].indexname "doublemapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "doublemapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "type double" @@ -196,6 +220,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "doublemapfield" indexinfo[].command[].command "type Map<int,double>" indexinfo[].command[].indexname "arraymapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "type string" @@ -232,10 +258,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructfield.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructfield.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "type string" @@ -254,6 +284,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "integer" @@ -268,10 +300,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructmap.value.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructmap.value.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "type string" @@ -288,6 +324,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.structfield" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap" indexinfo[].command[].command "multivalue" @@ -302,10 +340,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructarr.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructarr.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "type string" @@ -314,10 +356,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr.mymap" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mystructarr.title" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.structfield" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "type string" @@ -326,6 +372,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr" indexinfo[].command[].command "type Array<mystruct>" indexinfo[].command[].indexname "Folders.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "integer" @@ -342,10 +390,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -366,10 +418,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -390,10 +446,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -414,10 +474,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -438,10 +502,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -462,10 +530,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -486,10 +558,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -510,10 +586,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -534,10 +614,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -566,6 +650,8 @@ indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value.anotherfolder" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg index 3dd97e5c11f..0dc87b513ce 100644 --- a/config-model/src/test/derived/uri_array/ilscripts.cfg +++ b/config-model/src/test/derived/uri_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_array" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg index 48e07ef9959..cc45ee5ad8f 100644 --- a/config-model/src/test/derived/uri_wset/ilscripts.cfg +++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_wset" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index f9f74546a00..8a184e414d7 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -62,7 +62,7 @@ public class SystemModelTestCase { VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); assertNotNull(vespaModel); - assertEquals(4, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing and AdminModel (set up implicitly)"); + assertEquals(5, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing, Logs and AdminModel (set up implicitly)"); assertNotNull(vespaModel.configModelRepo().asMap().get("simple"), "One gets the default name as there is no explicit id"); assertNotNull(vespaModel.configModelRepo().asMap().get("second"), "The other gets the explicit id as name"); @@ -116,9 +116,9 @@ public class SystemModelTestCase { assertEquals(host1, host2); assertEquals(host2, host3); - // all three host aliases are for the same host, so the number of services should be 3 + 8 - // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver and metricsproxy) - assertEquals(10, host1.getServices().size()); + // all three host aliases are for the same host, so the number of services should be 3 + 9 + // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver, logserver-container and metricsproxy) + assertEquals(11, host1.getServices().size()); assertNotNull(host1.getService("simpleservice")); assertNotNull(host1.getService("simpleservice2")); @@ -145,7 +145,7 @@ public class SystemModelTestCase { assertNotNull(vespaModel); ApplicationConfigProducerRoot root = vespaModel.getVespa(); - assertEquals(5, vespaModel.configModelRepo().asMap().size()); + assertEquals(6, vespaModel.configModelRepo().asMap().size()); assertTrue(vespaModel.configModelRepo().asMap().containsKey("simple")); assertTrue(vespaModel.configModelRepo().asMap().containsKey("api")); assertTrue(root.getConfigIds().contains("simple/simpleservice.0")); @@ -155,6 +155,8 @@ public class SystemModelTestCase { // Verify that configModelRegistry iterates in dependency order Iterator<ConfigModel> i = vespaModel.configModelRepo().iterator(); ConfigModel plugin = i.next(); + assertEquals("logs", plugin.getId()); + plugin = i.next(); assertEquals("admin", plugin.getId()); plugin = i.next(); assertEquals("simple", plugin.getId()); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5ead9812b56..737ac4c248c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -60,7 +60,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.Host.memoryOverheadGb; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -151,7 +151,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().ofContainerAvailable()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -234,8 +234,8 @@ public class ModelProvisioningTest { assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(2, model.getContentClusters().get("content").getRootGroup().getNodes().size(), "Nodes in cluster without ID"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); - assertEquals(84, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -287,8 +287,8 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); - assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); + assertEquals(24, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); + assertEquals(1876900708, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); @@ -356,7 +356,7 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); assertEquals((long) ((3 - memoryOverheadGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is normal"); } @@ -2580,7 +2580,7 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) ((128 - memoryOverheadGb) * GB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) ((128 - memoryOverheadGb) * GiB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning } private static ProtonConfig getProtonConfig(VespaModel model, String configId) { diff --git a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java index df71e30c7d9..d9355e2df7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -3,49 +3,58 @@ package com.yahoo.schema; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * @author baldersheim */ public class DiversityTestCase { - @Test - void testDiversity() throws ParseException { + private static void verifyDiversity(DeployLoggerStub logger, boolean atRankProfile, boolean atMatchPhase) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(logger, rankProfileRegistry); + String diversitySpec = """ + diversity { + attribute: b + min-groups: 74 + cutoff-factor: 17.3 + cutoff-strategy: strict + } + """; builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int {\n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " cutoff-factor: 17.3\n" + - " cutoff-strategy: strict" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " max-filter-coverage: 0.065" + - " }\n" + - " }\n" + - "}\n"); + """ + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + } + field b type int { + indexing: attribute + } + } + rank-profile parent { + match-phase {""" + + (atMatchPhase ? diversitySpec : "") + + """ + attribute: a + max-hits: 120 + max-filter-coverage: 0.065 + }""" + + (atRankProfile ? diversitySpec : "") + + """ + } + } + """); builder.build(true); Schema s = builder.getSchema(); - RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); - RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + RankProfile parent = rankProfileRegistry.get(s, "parent"); + RankProfile.MatchPhaseSettings matchPhase = parent.getMatchPhase(); + RankProfile.DiversitySettings diversity = parent.getDiversity(); assertEquals("b", diversity.getAttribute()); assertEquals(74, diversity.getMinGroups()); assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); @@ -54,6 +63,44 @@ public class DiversityTestCase { assertEquals("a", matchPhase.getAttribute()); assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); } + @Test + void testDiversity() throws ParseException { + DeployLoggerStub logger = new DeployLoggerStub(); + verifyDiversity(logger, true, false); + assertTrue(logger.entries.isEmpty()); + verifyDiversity(logger, false, true); + assertEquals(1, logger.entries.size()); + assertEquals("'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level.", logger.entries.get(0).message); + try { + verifyDiversity(logger, true, true); + fail("Should throw."); + } catch (Exception e) { + assertEquals("rank-profile 'parent' error: already has diversity", e.getMessage()); + } + } + + @Test + void requireMatchPhaseOrSecondPhase() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field b type int { indexing: attribute } + } + rank-profile parent { + diversity { + attribute: b + min-groups: 74 + } + } + }"""); + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank-profile 'parent': 'diversity' requires either 'match-phase' or 'second-phase' to be specified.", e.getMessage()); + } + } private static String getMessagePrefix() { return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; @@ -82,30 +129,29 @@ public class DiversityTestCase { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - diversity + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " }\n" + - " }\n" + - "}\n"); + private ApplicationBuilder getSearchBuilder(String diversityField) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + }""" + + diversityField + + """ + } + rank-profile parent { + match-phase { + attribute: a + max-hits: 120 + } + diversity { + attribute: b + min-groups: 74 + } + } + }"""); return builder; } } diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 59887bfd57e..564bfd4a990 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -530,4 +530,28 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { "}"); } + @Test + public void secondPhaseRankScoreDropLimitIsAddedToRankProperties() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + String input = """ + schema test { + document test { + } + rank-profile test inherits default { + second-phase { + rank-score-drop-limit: 17.0 + } + } + } + """; + builder.addSchema(input); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(3, registry.all().size()); + RawRankProfile rawProfile = createRawRankProfile(registry.get(schema, "test"), schema); + assertEquals("17.0", findProperty(rawProfile.configProperties(), "vespa.hitcollector.secondphase.rankscoredroplimit").get()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 8ffbab84fd7..539ec91ee5f 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,16 +6,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -233,16 +230,12 @@ public class SummaryTestCase { " document-summary test_summary inherits nonesuch {" + " }" + "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document-summary 'test_summary' inherits 'nonesuch' but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); + ApplicationBuilder.createFromString(schema); + fail("Expected failure"); } catch (IllegalArgumentException e) { - fail(); - // assertEquals("document-summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); + assertEquals("document-summary 'test_summary' inherits 'nonesuch', but this is not present in schema 'test'", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java index 8d2c98439ca..af6fb82ee7b 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.parser; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.reader.NamedReader; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -232,5 +233,16 @@ public class IntermediateCollectionTestCase { assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); } + @Test + void can_detect_errors_in_rank_profile_outside_schema() { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + var exception = assertThrows(ParseException.class, () -> { + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity2/invalid_comment.profile"); + }); + var message = exception.getMessage(); + assertTrue(message.contains("Failed parsing rank-profile from 'src/test/derived/rankprofilemodularity2/invalid_comment.profile'")); + assertTrue(message.contains("Lexical error at line 3, column 6"), message); + } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index 5a2dc218da7..4186e352388 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -121,6 +121,39 @@ public class SchemaParserTestCase { } @Test + void significance_can_be_parsed() throws Exception { + String input = """ + schema foo { + rank-profile significance-ranking-0 inherits default { + significance { + use-model: true + } + } + rank-profile significance-ranking-1 { + significance { + use-model: false + } + } + } + """; + + ParsedSchema schema = parseString(input); + assertEquals("foo", schema.name()); + var rplist = schema.getRankProfiles(); + assertEquals(2, rplist.size()); + + var rp0 = rplist.get(0); + assertEquals("significance-ranking-0", rp0.name()); + assertTrue(rp0.isUseSignificanceModel().isPresent()); + assertTrue(rp0.isUseSignificanceModel().get()); + + var rp1 = rplist.get(1); + assertEquals("significance-ranking-1", rp1.name()); + assertTrue(rp1.isUseSignificanceModel().isPresent()); + assertFalse(rp1.isUseSignificanceModel().get()); + } + + @Test void maxOccurrencesCanBeParsed() throws Exception { String input = joinLines ("schema foo {", @@ -137,6 +170,23 @@ public class SchemaParserTestCase { assertEquals(11, field.matchSettings().getMaxTermOccurrences().get()); } + @Test + void maxTokenLengthCanBeParsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " field bar type string {", + " indexing: summary | index", + " match { max-token-length: 11 }", + " }", + " }", + "}"); + ParsedSchema schema = parseString(input); + var field = schema.getDocument().getFields().get(0); + assertEquals("bar", field.name()); + assertEquals(11, field.matchSettings().getMaxTokenLength().get()); + } + void checkFileParses(String fileName) throws Exception { var schema = parseFile(fileName); assertNotNull(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index de99d46b9ca..355a810f5ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.schema.Schema; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; @@ -155,6 +156,24 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field); } + @Test + void requireThatMaxTokenLengthIsPropagated() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-token-length:10 | summary test | index test; }", + field); + } + + @Test + void requireThatMaxTokenLengthIsPropagatedForWordMatch() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10).setType(MatchType.WORD); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | exact max-token-length:10 | summary test | index test; }", + field); + } + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { assertEquals(expectedScript, processField(unprocessedField).toString()); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java index 409e3b5df22..3f7c87ae37e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.derived.TestableDeployLogger; import com.yahoo.schema.parser.ParseException; import org.junit.jupiter.api.Test; @@ -90,6 +91,31 @@ public class PagedAttributeValidatorTestCase { assertPagedSupported("reference<parent>", Optional.of(getSd("parent", "int", false))); } + @Test + void hnsw_index_triggers_warning_with_paged_setting() throws ParseException { + var logger = new TestableDeployLogger(); + var sd = """ + schema test { + document test { + field pos type tensor(x[2]) { + indexing: attribute | index + attribute: paged + index { + hnsw { + } + } + } + } + } + """; + var schema = createFromString(sd, logger); + assertEquals(1, logger.warnings.size()); + assertEquals("For schema 'test', field 'pos': " + + "The 'paged' attribute setting in combination with HNSW indexing is strongly discouraged, see " + + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages for details", + logger.warnings.get(0)); + } + private void assertPagedSettingNotSupported(String fieldType) throws ParseException { assertPagedSettingNotSupported(fieldType, false); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index 9eca2106c5e..5ea097f13fb 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SummaryConsistencyTestCase { @@ -42,4 +43,24 @@ public class SummaryConsistencyTestCase { Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } + + @Test + void testDocumentSummaryWithInheritanceOfNonExistingSummary() { + String schemaString = """ + schema foo { + document foo { + field foo type string { + indexing: summary + } + } + document-summary foo_summary inherits non-existent { + summary foo { + source: foo + } + } + } + """; + assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString(schemaString).getSchema()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java index 4df9f261dfe..7aa6eb76995 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -258,7 +258,7 @@ public class ClusterInfoTest { .provisioned(provisioner.provisioned()) .build(); new VespaModel(new NullConfigModelRegistry(), deployState); - return deployState.provisioned().all(); + return deployState.provisioned().capacities(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index adf570c4664..96c6b1f99eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -50,8 +50,8 @@ public class AdminTestCase { void testAdmin20() { VespaModel vespaModel = getVespaModel(TESTDIR + "adminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -81,10 +81,10 @@ public class AdminTestCase { StateserverConfig.Builder ssb = new StateserverConfig.Builder(); vespaModel.getConfig(ssb, "admin/slobrok.0"); - assertEquals(19100, new StateserverConfig(ssb).httpport()); + assertEquals(19103, new StateserverConfig(ssb).httpport()); vespaModel.getConfig(ssb, "admin/slobrok.1"); - assertEquals(19102, new StateserverConfig(ssb).httpport()); + assertEquals(19105, new StateserverConfig(ssb).httpport()); LogdConfig.Builder lb = new LogdConfig.Builder(); vespaModel.getConfig(lb, "admin/slobrok.0"); @@ -98,12 +98,13 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(5, sentinelConfig.service().size()); + assertEquals(6, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals("slobrok2", sentinelConfig.service(2).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); - assertEquals("logd", sentinelConfig.service(4).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals("slobrok2", sentinelConfig.service(3).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(4).name()); + assertEquals("logd", sentinelConfig.service(5).name()); } /** @@ -114,8 +115,8 @@ public class AdminTestCase { void testOnlyAdminserver() { VespaModel vespaModel = getVespaModel(TESTDIR + "simpleadminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -134,11 +135,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(4, sentinelConfig.service().size()); + assertEquals(5, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(2).name()); - assertEquals("logd", sentinelConfig.service(3).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); + assertEquals("logd", sentinelConfig.service(4).name()); assertEquals(-1, sentinelConfig.service(0).affinity().cpuSocket()); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); @@ -175,8 +177,8 @@ public class AdminTestCase { void testMultipleConfigServers() { VespaModel vespaModel = getVespaModel(TESTDIR + "multipleconfigservers"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index ac431f081ed..0acacc340eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -150,7 +150,7 @@ public class MetricsProxyContainerTest { @Test void vespa_services_config_has_all_services() { VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(hostedServicesWithContent()); - assertEquals(10, vespaServicesConfig.service().size()); + assertEquals(11, vespaServicesConfig.service().size()); for (var service : vespaServicesConfig.service()) { if (service.configId().equals("admin/cluster-controllers/0")) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index 7c4968aac84..efe9cfe5060 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -2,8 +2,24 @@ package com.yahoo.vespa.model.admin.otel; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; /** @@ -13,10 +29,66 @@ public class OpenTelemetryConfigGeneratorTest { @Test void testBuildsYaml() { - var generator = new OpenTelemetryConfigGenerator(null); - generator.addStatePorts(List.of(new StatePortInfo("localhost", 19098, "config-sentinel", "sentinel"))); + var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); + var app = ApplicationId.from("mytenant", "myapp", "myinstance"); + var generator = new OpenTelemetryConfigGenerator(mockZone, app, true); + var root = new MockRoot(); + + var mockHost = new Host(root, "localhost2.local"); + var mockVersion = new com.yahoo.component.Version(8); + var mockCluster = ClusterMembership.from("container/feeding/2/3", mockVersion, Optional.empty()); + var noResource = NodeResources.unspecified(); + var mockHostSpec = new HostSpec("localhost1.local", + noResource, noResource, noResource, + mockCluster, + Optional.empty(), Optional.empty(), Optional.empty()); + var mockHostResource = new HostResource(mockHost, mockHostSpec); + var mockSvc1 = new MockService(root, "sentinel"); + mockSvc1.setHostResource(mockHostResource); + var mockPort1 = new StatePortInfo("localhost", 19098, mockSvc1); + + var mockSvc2 = new MockService(root, "searchnode"); + mockSvc2.setProp("clustername", "mycluster"); + mockSvc2.setProp("clustertype", "mockup"); + var mockPort2 = new StatePortInfo("other123x.host.local", 19102, mockSvc2); + + generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); + // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); + String want = """ + "parentHostname":"other123.host.local"""; + assertTrue(yaml.contains(want)); } + static class MockService extends AbstractService { + private final String name; + public MockService(TreeConfigProducer<?> parent, String name) { + super(parent, name); + this.name = name; + } + public String getServiceName() { return name; } + public String getServiceType() { return "dummy"; } + @Override public int getPortCount() { return 0; } + @Override public void allocatePorts(int start, PortAllocBridge from) { } + } + + @Test + void testFindParentHost() { + String result; + result = OpenTelemetryConfigGenerator.findParentHost("n1234c.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("n1234-v6-7.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000a.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000-v6-10.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("foobar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123bar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123.some.cloud"); + assertNull(result); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index ae1db366c9f..2e51a425f6d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -143,6 +143,32 @@ public class ComplexFieldsValidatorTestCase { } @Test + void logs_warning_when_complex_fields_have_struct_fields_with_index_and_exact_match() throws IOException, SAXException { + var logger = new MyLogger(); + createModelAndValidate(joinLines( + "schema test {", + " document test {", + " field nesteds type array<nested> {", + " struct-field foo {", + " indexing: attribute | index", + " match {", + " exact", + " exact-terminator: '@@'", + " }", + " }", + " }", + " struct nested {", + " field foo type string {}", + " }", + " }", + "}"), logger); + assertTrue(logger.message.toString().contains("For cluster 'mycluster', schema 'test': " + + "The following complex fields have struct fields with 'indexing: index' which is " + + "not supported and has no effect: nesteds (nesteds.foo). " + + "Remove setting or change to 'indexing: attribute' if needed for matching.")); + } + + @Test void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException { createModelAndValidate(joinLines("search test {", " document test {", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 45125c8eb68..340968f89d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -45,10 +45,10 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.3, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, - "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); + "Allocated memory to JVM in cluster 'container' is too low (0.51GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 89f81dfdaef..590433757c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -54,6 +54,12 @@ public class QuotaValidatorTest { } @Test + void test_deploy_within_quota_budget_because_in_dev() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(devZone)); + tester.deploy(null, getServices(10), Environment.dev, null, CONTAINER_CLUSTER); + } + + @Test void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index f68a1da7dfb..0f7113748dd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -57,10 +57,11 @@ public class ConfigValueChangeValidatorTest { createVespaModel(createQrStartConfigSegment(true, 2096)), createVespaModel(createQrStartConfigSegment(false, 2096)) ); - assertEquals(3, changes.size()); + assertEquals(4, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); - assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "admin/metrics/localhost", 2); + assertComponentsEquals(changes, "admin/logs/0", 1); + assertComponentsEquals(changes, "admin/cluster-controllers/0", 2); + assertComponentsEquals(changes, "admin/metrics/localhost", 3); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index b7e612127c1..36839e72e10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -209,7 +209,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <container id='default' version='1.0'>" + " <nodes count='" + nodes + "'>" + @@ -223,7 +223,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java new file mode 100644 index 00000000000..13e91f60712 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidatorTest { + + private static final String LOCAL_LLM_COMPONENT = RestartOnDeployForLocalLLMValidator.LOCAL_LLM_COMPONENT; + + @Test + void validate_no_restart_on_deploy() { + VespaModel current = createModel(); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(0, result.size()); + } + + @Test + void validate_restart_on_deploy() { + VespaModel current = createModel(withComponent(LOCAL_LLM_COMPONENT)); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(1, result.size()); + assertTrue(result.get(0).validationId().isEmpty()); + assertEquals("Need to restart services in cluster 'cluster1' due to use of local LLM", result.get(0).getMessage()); + } + + private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { + return ValidationTester.validateChanges(new RestartOnDeployForLocalLLMValidator(), + next, + deployStateBuilder().previousModel(current).build()); + } + + private static VespaModel createModel(String component) { + var xml = """ + <services version='1.0'> + <container id='cluster1' version='1.0'> + <http> + <server id='server1' port='8080'/> + </http> + %s + </container> + </services> + """.formatted(component); + DeployState.Builder builder = deployStateBuilder(); + return new VespaModelCreatorWithMockPkg(null, xml).create(builder); + } + + private static VespaModel createModel() { + return createModel(""); + } + + private static String withComponent(String componentClass) { + return "<component id='llm' class='%s' />".formatted(componentClass); + } + + private static DeployState.Builder deployStateBuilder() { + return new DeployState.Builder().properties(new TestProperties()); + } + + private static void assertStartsWith(String expected, List<ConfigChangeAction> result) { + assertTrue(result.get(0).getMessage().startsWith(expected)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 4c0786ea879..484a938476d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -33,7 +33,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { // Must be so large that changing model set or options requires restart (due to using more memory than available), // but not so large that deployment will not work at all with one model - private static final long defaultCost = 723456789; + private static final long defaultCost = 635241309; private static final long defaultHash = 0; @@ -47,8 +47,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost() { - VespaModel current = createModel(onnxModelCost(70000000, defaultHash)); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash)); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash)); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash)); List<ConfigChangeAction> result = validateModel(current, next); assertEquals(1, result.size()); assertTrue(result.get(0).validationId().isEmpty()); @@ -58,8 +58,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost_non_hosted() { boolean hosted = false; - VespaModel current = createModel(onnxModelCost(70000000, defaultHash), hosted); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash), hosted); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash), hosted); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash), hosted); List<ConfigChangeAction> result = validateModel(current, next, hosted); assertEquals(0, result.size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 8778f0c26c0..0677cabafb0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -29,11 +29,18 @@ public abstract class ContentClusterFixture { nextCluster = createCluster(nextSd); } + protected ContentClusterFixture(ContentCluster currentCluster, ContentCluster nextCluster) { + this.currentCluster = currentCluster; + this.nextCluster = nextCluster; + } + public ContentClusterFixture(String entireSd) throws Exception { - currentCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); - nextCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); + currentCluster = createClusterFromEntireSd(entireSd); + nextCluster = createClusterFromEntireSd(entireSd); + } + + protected static ContentCluster createClusterFromEntireSd(String sdContent) throws Exception { + return new ContentClusterBuilder().build(ContentClusterUtils.createMockRoot(List.of(sdContent))); } private static ContentCluster createCluster(String sdContent) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index cd54a20523f..247f01068fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -39,6 +39,21 @@ public class IndexingScriptChangeValidatorTest { } } + private static class ComplexFixture extends ContentClusterFixture { + IndexingScriptChangeValidator validator; + public ComplexFixture(String currentSd, String nextSd) throws Exception { + super(createClusterFromEntireSd(currentSd), createClusterFromEntireSd(nextSd)); + validator = new IndexingScriptChangeValidator(ClusterSpec.Id.from("test"), + currentDb().getDerivedConfiguration().getSchema(), + nextDb().getDerivedConfiguration().getSchema()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(); + } + } + private static class ScriptFixture { private final ScriptExpression currentScript; @@ -56,6 +71,9 @@ public class IndexingScriptChangeValidatorTest { private static final String FIELD = "field f1 type string"; private static final String FIELD_F2 = "field f2 type string"; + private static final String TENSOR_FIELD_F1 = "field f1 type tensor(x[2])"; + private static final String TENSOR_FIELD_F2 = "field f2 type tensor(x[2])"; + private static final String TENSOR_FIELD_F3 = "field f3 type tensor(x[2])"; private static VespaConfigChangeAction expectedReindexingAction(String changedMsg, String fromScript, String toScript) { return expectedReindexingAction("f1", changedMsg, fromScript, toScript); @@ -115,6 +133,28 @@ public class IndexingScriptChangeValidatorTest { } @Test + void requireThatAddingIndexAspectForExtraTensorFieldWithChangedInputRequireReindexing() throws Exception { + new ComplexFixture(joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f1 | attribute }", + "}"), + joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f2 | index | attribute }", + "}")). + assertValidation(List.of(expectedReindexingAction("f3", "add index aspect", + "{ input f1 | attribute f3; }", + "{ input f2 | index f3 | attribute f3; }"))); + } + + + @Test void requireThatSettingDynamicSummaryIsOk() throws Exception { new Fixture(FIELD + " { indexing: summary }", FIELD + " { indexing: summary \n summary: dynamic }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 924419daeae..052d32269ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -157,7 +157,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logserver-container", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); @@ -205,7 +205,7 @@ public class ContentBuilderTest extends DomBuilderTest { HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", - "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", + "logserver-container", "slobrok", "storagenode", "distributor", "searchnode", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 0e616661191..feeff452a91 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; @@ -196,14 +197,21 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { assertEquals(1, admin.getConfigservers().size()); } + @Test + void containerOnLogserver() { + Admin admin = buildAdmin(servicesAdminNoAdminServerOrConfigServer()); + assertTrue(admin.getLogServerContainerCluster().isPresent()); + assertEquals("logs", admin.getLogServerContainerCluster().get().getName()); + } + private Admin buildAdmin(Element xml) { return buildAdmin(xml, false, new ArrayList<>()); } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { DeployState deployState = DeployState.createTestState(); - final DomAdminV2Builder domAdminBuilder = - new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, multitenant, configServerSpecs); + ConfigModelContext context = ConfigModelContext.create(deployState, null, (m) -> {}, root, "foo"); + DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(context, multitenant, configServerSpecs); Admin admin = domAdminBuilder.build(deployState, root, xml); admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java index 344471cada0..e0193d10e23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java @@ -42,8 +42,8 @@ public class NodesSpecificationTest { assertEquals(2, spec.minResources().nodeResources().vcpu(), 1e-9); assertEquals(2, spec.maxResources().nodeResources().vcpu(), 1e-9); - assertEquals(3e15, spec.minResources().nodeResources().memoryGb(), 1e-9); - assertEquals(3e15, spec.maxResources().nodeResources().memoryGb(), 1e-9); + assertEquals(3e15, spec.minResources().nodeResources().memoryGiB(), 1e-9); + assertEquals(3e15, spec.maxResources().nodeResources().memoryGiB(), 1e-9); assertEquals(4e3, spec.minResources().nodeResources().diskGb(), 1e-9); assertEquals(4e3, spec.maxResources().nodeResources().diskGb(), 1e-9); @@ -54,8 +54,8 @@ public class NodesSpecificationTest { assertEquals(1 << 30, spec.minResources().nodeResources().gpuResources().count()); assertEquals(1 << 30, spec.maxResources().nodeResources().gpuResources().count()); - assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGb(), 1e-12); - assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGb(), 1e-12); + assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGiB(), 1e-12); + assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGiB(), 1e-12); assertEquals(DiskSpeed.fast, spec.minResources().nodeResources().diskSpeed()); assertEquals(DiskSpeed.fast, spec.maxResources().nodeResources().diskSpeed()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2ea4249883d..93362e144a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -49,6 +49,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -221,24 +222,6 @@ public class ContainerClusterTest { } @Test - void requireThatJvmOmitStackTraceInFastThrowOptionWorks() { - // Empty option if option not set in property - MockRoot root = createRoot(new DeployState.Builder().build()); - ApplicationContainerCluster cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - ApplicationContainer container = cluster.getContainers().get(0); - assertEquals("", container.getJvmOptions()); - - String jvmOption = "-XX:-foo"; - DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build(); - root = createRoot(deployState); - cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - container = cluster.getContainers().get(0); - assertEquals(jvmOption, container.getJvmOptions()); - } - - @Test void requireThatWeCanHandleNullJvmOptions() { MockRoot root = createRoot(false); ApplicationContainerCluster cluster = newClusterWithSearch(root); @@ -516,4 +499,28 @@ public class ContainerClusterTest { (extra.isEmpty() ? "" : " " + extra); } + @Test + void testTruncationTo4Bits() { + assertEquals(0, ApplicationContainerCluster.truncateTo4SignificantBits(0)); + assertEquals(1, ApplicationContainerCluster.truncateTo4SignificantBits(1)); + assertEquals(15, ApplicationContainerCluster.truncateTo4SignificantBits(15)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(16)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(17)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(18)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(19)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(30)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(31)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(32)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(33)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(34)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(35)); + assertEquals(36, ApplicationContainerCluster.truncateTo4SignificantBits(36)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x78000000)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x7fffffff)); + assertEquals(0x80000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x80000000)); + assertEquals(0b10001000000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b10000000000000000000000000000001)); + assertEquals(0b11000100000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b11000000000000000000000000000001)); + assertEquals(0b11111111111111111111111111100010, ApplicationContainerCluster.truncateTo4SignificantBits(0b11111111111111111111111111100001)); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index ea43f5c8124..b782366655f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertTrue(chain.phases().isEmpty()); assertEquals(1, chain.inherits().size()); assertEquals("native", chain.inherits(0)); - assertEquals(10, chain.components().size()); + assertEquals(11, chain.components().size()); assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0)); assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1)); assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2)); @@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7)); assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8)); assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9)); + assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10)); assertTrue(chain.excludes().isEmpty()); assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type()); } @Test public void require_all_default_chains_are_correct() { - assertEquals(63, chainsConfig.components().size()); + assertEquals(64, chainsConfig.components().size()); assertEquals(10, chainsConfig.chains().size()); validateVespaPhasesChain(findChain("vespaPhases")); validateNativeChain(findChain("native")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e704da08d18..60ea37cad3f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -49,6 +50,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -598,7 +600,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setCloudAccount(cloudAccount)) .build()); assertEquals(2, model.hostSystem().getHosts().size()); - assertEquals(List.of(cloudAccount), model.provisioned().all().values() + assertEquals(List.of(cloudAccount), model.provisioned().capacities().values() .stream() .map(capacity -> capacity.cloudAccount().get()) .toList()); @@ -741,6 +743,29 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void testerContainer() { + createModelWithTesterNodes("<nodes count='1' docker-image='foo/bar/baz'><resources vcpu='0.1' memory='1Gb' disk='1Gb'/></nodes>"); + + assertEquals("In container cluster 'default': tester cannot run on more than 1 node, but 2 nodes were specified", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes count='2'/>")))); + + assertEquals("In container cluster 'default': tester resources must be absolute, but min and max resources differ: specification of dedicated " + + "min 1 nodes with [vcpu: 0.0, memory: 1.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any] " + + "max 1 nodes with [vcpu: 0.0, memory: 2.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes><resources memory='[1Gb, 2Gb]'/></nodes>")))); + } + + void createModelWithTesterNodes(String testerNodesXml) { + String containerXml = "<container id='default' version='1.0'>%s</container>".formatted(testerNodesXml); + VespaModelTester tester = new VespaModelTester(); + tester.setApplicationId("t", "a", "i-t"); + tester.addHosts(3); + tester.createModel(containerXml, true); + } + + @Test void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 786caa4b317..a890f35caa8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; @@ -48,6 +49,8 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -941,8 +944,19 @@ public class ContentClusterTest extends ContentBaseTest { } private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(clusterXml, props, flavor, new StringBuffer()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, + Optional<Flavor> flavor, StringBuffer deployWarningsBuffer) throws Exception { + DeployLogger logger = (level, message) -> { + if (level == Level.WARNING) { // only care about warnings + deployWarningsBuffer.append("%s\n".formatted(message)); + } + }; DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(props); + .properties(props) + .deployLogger(logger); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), List.of(), deployStateBuilder) : @@ -1264,7 +1278,7 @@ public class ContentClusterTest extends ContentBaseTest { void verify_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); + assertEquals(2_000_000_000, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, @@ -1471,13 +1485,18 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); } - private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + private StorDistributormanagerConfig resolveDistributorConfig(Consumer<TestProperties> propertyMutator) throws Exception { var properties = new TestProperties(); - if (featureLevel != null) { - properties.setContentLayerMetadataFeatureLevel(featureLevel); - } - var cfg = resolveStorDistributormanagerConfig(properties); - return cfg.enable_operation_cancellation(); + propertyMutator.accept(properties); + return resolveStorDistributormanagerConfig(properties); + } + + private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + return resolveDistributorConfig((props) -> { + if (featureLevel != null) { + props.setContentLayerMetadataFeatureLevel(featureLevel); + } + }).enable_operation_cancellation(); } @Test @@ -1488,6 +1507,21 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + private boolean resolveDistributorSymmetricReplicaSelectionConfig(Boolean flagValue) throws Exception { + return resolveDistributorConfig((props) -> { + if (flagValue != null) { + props.setSymmetricPutAndActivateReplicaSelection(flagValue); + } + }).symmetric_put_and_activate_replica_selection(); + } + + @Test + void distributor_symmetric_replica_selection_config_controlled_by_properties() throws Exception { + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(null)); // defaults to false + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(false)); + assertTrue(resolveDistributorSymmetricReplicaSelectionConfig(true)); + } + @Test void node_distribution_key_outside_legal_range_is_disallowed() { // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal @@ -1506,6 +1540,68 @@ public class ContentClusterTest extends ContentBaseTest { } } + private String createClusterAndGetDeploymentWarnings(String xml) { + var warningBuf = new StringBuffer(); + try { + createOneNodeCluster(xml, new TestProperties(), Optional.empty(), warningBuf); + } catch (Exception e) { + throw new RuntimeException(e); + } + return warningBuf.toString(); + }; + + private static String clusterXmlWithNodeDistributionKey(int key) { + return "<content version=\"1.0\" id=\"mockcluster\">" + + " <redundancy>1</redundancy>" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"%d\" hostalias=\"mockhost\"/>".formatted(key) + + " </group>" + + "</content>"; + } + + @Test + void distribution_key_much_higher_than_node_count_logs_deployment_warning() { + // "much higher" is somewhat arbitrary, but needs to be kept in track with threshold in `ContentCluster`. + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(101)); + assertEquals(warnings, "Content cluster 'mockcluster' has 1 node(s), but the highest distribution " + + "key is 101. Having much higher distribution keys than the number of nodes " + + "is not recommended, as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node\n"); + } + + @Test + void distribution_key_not_much_higher_than_node_count_does_not_log_deployment_warning() { + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(100)); + assertEquals(warnings, ""); + } + + private void checkStrictlyIncreasingClusterStateVersionConfig(Boolean flagValue, boolean expected) throws Exception { + var props = new TestProperties(); + if (flagValue != null) { + props.setEnforceStrictlyIncreasingClusterStateVersions(flagValue); + } + var cc = createOneNodeCluster(props); + + // stor-server config should be the same for both distributors and storage nodes + var builder = new StorServerConfig.Builder(); + cc.getStorageCluster().getConfig(builder); + var cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + + builder = new StorServerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + } + + @Test + void strictly_increasing_cluster_state_versions_config_controlled_by_feature_flag() throws Exception { + checkStrictlyIncreasingClusterStateVersionConfig(null, false); // TODO change default + checkStrictlyIncreasingClusterStateVersionConfig(false, false); + checkStrictlyIncreasingClusterStateVersionConfig(true, true); + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index e9e96d8b0cf..920b2f7a868 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -131,7 +131,7 @@ public class IndexedTest extends ContentBaseTest { // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", // "container", "fleetcontroller", - // "storagenode", "searchnode", "distributor", "transactionlogserver"}; + // "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); assertNotNull(routing); @@ -159,8 +159,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "container", "storagenode", "searchnode", "distributor", - // "transactionlogserver"}; + // "container", "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 8b83c941631..2a81f2da286 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -111,9 +111,9 @@ public class ClusterTest { assertEquals(1, nodesConfig.node(1).key()); assertEquals(2, nodesConfig.node(2).key()); - assertEquals(19106, nodesConfig.node(0).port()); - assertEquals(19118, nodesConfig.node(1).port()); - assertEquals(19130, nodesConfig.node(2).port()); + assertEquals(19109, nodesConfig.node(0).port()); + assertEquals(19121, nodesConfig.node(1).port()); + assertEquals(19133, nodesConfig.node(2).port()); assertEquals(0, nodesConfig.node(0).group()); assertEquals(0, nodesConfig.node(1).group()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index 4fd61f59ed7..6114cc7d8bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -8,9 +8,14 @@ import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.ApplicationPackageTester; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelCost.Calculator; +import com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; @@ -22,7 +27,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -47,6 +55,7 @@ public class ImportedModelTester { private final String modelName; private final Path applicationDir; private final DeployState deployState; + public final Calculator calculator = new MockCalculator(); public ImportedModelTester(String modelName, Path applicationDir) { this(modelName, applicationDir, new DeployState.Builder()); @@ -58,6 +67,7 @@ public class ImportedModelTester { deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) + .onnxModelCost((pkg, app, cluster) -> calculator) .build(); } @@ -98,4 +108,19 @@ public class ImportedModelTester { } } + public static class MockCalculator implements OnnxModelCost.Calculator { + private final Map<String, ModelInfo> models = new HashMap<>(); + @Override public long aggregatedModelCostInBytes() { return models.size(); } + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) { + models.put(path.toString(), new ModelInfo(path.toString(), 1, 1, onnxModelOptions)); + } + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + models.put(uri.toString(), new ModelInfo(uri.toString(), 1, 1, onnxModelOptions)); + } + @Override public Map<String, ModelInfo> models() { return Map.copyOf(models); } + @Override public void setRestartOnDeploy() { } + @Override public boolean restartOnDeploy() { return false; } + @Override public void store() { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index cc33c8561fc..bced4c546c6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -55,6 +55,7 @@ public class ModelEvaluationTest { RankProfilesConfig config = new RankProfilesConfig(b); assertEquals(0, config.rankprofile().size()); + assertEquals(0, tester.calculator.aggregatedModelCostInBytes()); } finally { IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); @@ -69,6 +70,7 @@ public class ModelEvaluationTest { try { ImportedModelTester tester = new ImportedModelTester("ml_serving", appDir); assertHasMlModels(tester.createVespaModel(), appDir); + assertEquals(3, tester.calculator.aggregatedModelCostInBytes()); // At this point the expression is stored - copy application to another location which do not have a models dir storedAppDir.toFile().mkdirs(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index fbc4f6768a2..998437b64f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -9,7 +9,8 @@ import org.junit.jupiter.api.Test; import static com.yahoo.vespa.model.Host.memoryOverheadGb; import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.MiB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; /** @@ -28,7 +29,7 @@ public class NodeResourcesTuningTest { @Test void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); + assertEquals(24 * GiB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); assertEquals(1.9585050869E10, configFromMemorySetting(24 + memoryOverheadGb, ApplicationContainerCluster.heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster * 0.01).hwinfo().memory().size(), 1000); } @@ -84,26 +85,26 @@ public class NodeResourcesTuningTest { @Test void require_that_document_store_maxfilesize_is_set_based_on_available_memory() { - assertDocumentStoreMaxFileSize(256 * MB, 4); - assertDocumentStoreMaxFileSize(256 * MB, 6); - assertDocumentStoreMaxFileSize(256 * MB, 8); - assertDocumentStoreMaxFileSize(256 * MB, 12); - assertDocumentStoreMaxFileSize((long) (16 * GB * 0.02), 16); - assertDocumentStoreMaxFileSize((long) (24 * GB * 0.02), 24); - assertDocumentStoreMaxFileSize((long) (32 * GB * 0.02), 32); - assertDocumentStoreMaxFileSize((long) (48 * GB * 0.02), 48); - assertDocumentStoreMaxFileSize((long) (64 * GB * 0.02), 64); - assertDocumentStoreMaxFileSize((long) (128 * GB * 0.02), 128); - assertDocumentStoreMaxFileSize((long) (256 * GB * 0.02), 256); - assertDocumentStoreMaxFileSize((long) (512 * GB * 0.02), 512); + assertDocumentStoreMaxFileSize(256 * MiB, 4); + assertDocumentStoreMaxFileSize(256 * MiB, 6); + assertDocumentStoreMaxFileSize(256 * MiB, 8); + assertDocumentStoreMaxFileSize(256 * MiB, 12); + assertDocumentStoreMaxFileSize((long) (16 * GiB * 0.02), 16); + assertDocumentStoreMaxFileSize((long) (24 * GiB * 0.02), 24); + assertDocumentStoreMaxFileSize((long) (32 * GiB * 0.02), 32); + assertDocumentStoreMaxFileSize((long) (48 * GiB * 0.02), 48); + assertDocumentStoreMaxFileSize((long) (64 * GiB * 0.02), 64); + assertDocumentStoreMaxFileSize((long) (128 * GiB * 0.02), 128); + assertDocumentStoreMaxFileSize((long) (256 * GiB * 0.02), 256); + assertDocumentStoreMaxFileSize((long) (512 * GiB * 0.02), 512); } @Test void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { - assertFlushStrategyMemory((long) (4 * GB * DEFAULT_MEMORY_GAIN), 4); - assertFlushStrategyMemory((long) (8 * GB * DEFAULT_MEMORY_GAIN), 8); - assertFlushStrategyMemory((long) (24 * GB * DEFAULT_MEMORY_GAIN), 24); - assertFlushStrategyMemory((long) (64 * GB * DEFAULT_MEMORY_GAIN), 64); + assertFlushStrategyMemory((long) (4 * GiB * DEFAULT_MEMORY_GAIN), 4); + assertFlushStrategyMemory((long) (8 * GiB * DEFAULT_MEMORY_GAIN), 8); + assertFlushStrategyMemory((long) (24 * GiB * DEFAULT_MEMORY_GAIN), 24); + assertFlushStrategyMemory((long) (64 * GiB * DEFAULT_MEMORY_GAIN), 64); } @Test @@ -129,8 +130,8 @@ public class NodeResourcesTuningTest { @Test void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1 * GB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); - assertEquals(256 * GB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(1 * GiB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(256 * GiB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index bc981c3de7c..499df4e5669 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,13 +30,8 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { + private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); - tls.setHostResource(new HostResource(host)); - tls.setBasePort(100); - tls.initService(root.getDeployState()); - node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); node.initService(root.getDeployState()); @@ -45,13 +39,15 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, - boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { + boolean flushOnShutDown, boolean isHosted, + ModelContext.FeatureFlags featureFlags, + Boolean syncTransactionLog) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - null, null, isHosted, 0.0, featureFlags); + null, null, isHosted, 0.0, featureFlags, syncTransactionLog); } - private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties()); + private static SearchNode createSearchNode(MockRoot root, Boolean syncTransactionLog) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties(), syncTransactionLog); } @Test @@ -64,15 +60,17 @@ public class SearchNodeTest { @Test void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); - prepare(root, node, true); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); + prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @Test void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -81,7 +79,8 @@ public class SearchNodeTest { @Test void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); @@ -90,7 +89,8 @@ public class SearchNodeTest { private void verifyCodePlacement(boolean hugePages) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().loadCodeAsHugePages(hugePages)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().loadCodeAsHugePages(hugePages), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(hugePages, node.getEnvVars().get("VESPA_LOAD_CODE_AS_HUGEPAGES") != null); @@ -104,7 +104,8 @@ public class SearchNodeTest { private void verifySharedStringRepoReclaim(boolean sharedStringRepoNoReclaim) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(sharedStringRepoNoReclaim, node.getEnvVars().get("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != null); @@ -120,10 +121,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean syncTransactionLog) { MockRoot root = createRoot(properties); - SearchNode node = createSearchNode(root); - prepare(root, node, useFsync); + SearchNode node = createSearchNode(root, syncTransactionLog); + prepare(root, node); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java index 00e95a34287..26e8c67a226 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java @@ -37,9 +37,6 @@ public class SignificanceModelTestCase { ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); var significanceConfig = assertSignificancePresent(containerCluster); assertEquals(3, significanceConfig.model().size()); - assertEquals("en", significanceConfig.model().get(0).language()); - assertEquals("no", significanceConfig.model().get(1).language()); - assertEquals("ru", significanceConfig.model().get(2).language()); assertEquals("models/idf-norwegian-wiki.json.zst", modelReference(significanceConfig.model().get(1), "path").path().orElseThrow().value()); assertEquals("https://some/uri/blob.json", modelReference(significanceConfig.model().get(2), "path").url().orElseThrow().value()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index c7d9ae6b818..2fda76d0d82 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -20,9 +20,6 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; @@ -37,6 +34,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; + import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -265,9 +263,9 @@ public class VespaModelTestCase { assertEquals(1, admin.getConfigservers().size()); Set<HostInfo> hosts = model.getHosts(); assertEquals(1, hosts.size()); - //logd, config proxy, sentinel, config server, slobrok, log server + // logd, config proxy, sentinel, config server, slobrok, logserver, logserver container HostInfo host = hosts.iterator().next(); - assertEquals(7, host.getServices().size()); + assertEquals(8, host.getServices().size()); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 7037480f853..887db9c9cc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -88,7 +88,7 @@ public class VespaModelTester { // (for e.g cluster controllers and slobrok nodes) String hostname = String.format("%s-%02d", "node" + "-" + Math.round(resources.vcpu()) + - "-" + Math.round(resources.memoryGb()) + + "-" + Math.round(resources.memoryGiB()) + "-" + Math.round(resources.diskGb()), count - i); hosts.add(new Host(hostname, List.of(), flavor)); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 7333ef5a87b..a413ec7753b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -168,7 +168,7 @@ </threadpool> <significance> - <model language="en" model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> + <model model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> </significance> </search> |