diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/schema')
22 files changed, 203 insertions, 145 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 82ed45028b3..ed1a4e98b49 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -73,7 +73,8 @@ public class RankProfile implements Cloneable { /** The resolved inherited profiles, or null when not resolved. */ private List<RankProfile> inherited; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); @@ -106,6 +107,7 @@ public class RankProfile implements Cloneable { /** The drop limit used to drop hits with rank score less than or equal to this value */ private double rankScoreDropLimit = -Double.MAX_VALUE; + private double secondPhaseRankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; private String inheritedSummaryFeaturesProfileName; @@ -141,6 +143,8 @@ public class RankProfile implements Cloneable { private Boolean strict; + private Boolean useSignificanceModel; + private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -216,6 +220,16 @@ public class RankProfile implements Cloneable { this.strict = strict; } + public void setUseSignificanceModel(Boolean useSignificanceModel) { + this.useSignificanceModel = useSignificanceModel; + } + + public boolean useSignificanceModel() { + if (useSignificanceModel != null) return useSignificanceModel; + return uniquelyInherited(RankProfile::useSignificanceModel, "use-model") + .orElse(false); // Disabled by default + } + /** * Adds a profile to those inherited by this. * The profile must belong to this schema (directly or by inheritance). @@ -294,20 +308,28 @@ public class RankProfile implements Cloneable { return false; } - public void setMatchPhaseSettings(MatchPhaseSettings settings) { + public void setMatchPhase(MatchPhaseSettings settings) { settings.checkValid(); - this.matchPhaseSettings = settings; + this.matchPhase = settings; } - public MatchPhaseSettings getMatchPhaseSettings() { - if (matchPhaseSettings != null) return matchPhaseSettings; - return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); + public MatchPhaseSettings getMatchPhase() { + if (matchPhase != null) return matchPhase; + return uniquelyInherited(RankProfile::getMatchPhase, "match phase settings").orElse(null); + } + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + public DiversitySettings getDiversity() { + if (diversity != null) return diversity; + return uniquelyInherited(RankProfile::getDiversity, "diversity settings").orElse(null); } /** Returns the uniquely determined property, where non-empty is defined as non-null */ private <T> Optional<T> uniquelyInherited(Function<RankProfile, T> propertyRetriever, String propertyDescription) { - return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); + return uniquelyInherited(propertyRetriever, Objects::nonNull, propertyDescription); } /** @@ -322,8 +344,8 @@ public class RankProfile implements Cloneable { Predicate<T> nonEmptyValueFilter, String propertyDescription) { Set<T> uniqueProperties = inherited().stream() - .map(p -> propertyRetriever.apply(p)) - .filter(p -> nonEmptyValueFilter.test(p)) + .map(propertyRetriever) + .filter(nonEmptyValueFilter) .collect(Collectors.toSet()); if (uniqueProperties.isEmpty()) return Optional.empty(); if (uniqueProperties.size() == 1) return uniqueProperties.stream().findAny(); @@ -482,7 +504,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getFirstPhase, "first-phase expression").orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -509,7 +531,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getSecondPhase, "second-phase expression").orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -529,7 +551,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getGlobalPhase() { if (globalPhaseRanking != null) return globalPhaseRanking; - return uniquelyInherited(p -> p.getGlobalPhase(), "global-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getGlobalPhase, "global-phase expression").orElse(null); } public void setGlobalPhaseRanking(String expression) { @@ -588,7 +610,7 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") + return uniquelyInherited(RankProfile::getSummaryFeatures, f -> ! f.isEmpty(), "summary features") .orElse(Set.of()); } @@ -605,13 +627,13 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") + return uniquelyInherited(RankProfile::getMatchFeatures, f -> ! f.isEmpty(), "match features") .orElse(Set.of()); } public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); - return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") + return uniquelyInherited(RankProfile::getHiddenMatchFeatures, f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); } @@ -649,7 +671,7 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") + return uniquelyInherited(RankProfile::getRankFeatures, f -> ! f.isEmpty(), "summary-features") .orElse(Set.of()); } @@ -680,7 +702,7 @@ public class RankProfile implements Cloneable { if (rankProperties.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); - var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") + var inheritedProperties = uniquelyInherited(RankProfile::getRankPropertyMap, m -> ! m.isEmpty(), "rank-properties") .orElse(Map.of()); if (rankProperties.isEmpty()) return inheritedProperties; @@ -722,21 +744,21 @@ public class RankProfile implements Cloneable { public int getRerankCount() { if (rerankCount >= 0) return rerankCount; - return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getRerankCount, c -> c >= 0, "rerank-count").orElse(-1); } public void setGlobalPhaseRerankCount(int count) { this.globalPhaseRerankCount = count; } public int getGlobalPhaseRerankCount() { if (globalPhaseRerankCount >= 0) return globalPhaseRerankCount; - return uniquelyInherited(p -> p.getGlobalPhaseRerankCount(), c -> c >= 0, "global-phase rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getGlobalPhaseRerankCount, c -> c >= 0, "global-phase rerank-count").orElse(-1); } public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } public int getNumThreadsPerSearch() { if (numThreadsPerSearch >= 0) return numThreadsPerSearch; - return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") + return uniquelyInherited(RankProfile::getNumThreadsPerSearch, n -> n >= 0, "num-threads-per-search") .orElse(-1); } @@ -744,14 +766,14 @@ public class RankProfile implements Cloneable { public int getMinHitsPerThread() { if (minHitsPerThread >= 0) return minHitsPerThread; - return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); + return uniquelyInherited(RankProfile::getMinHitsPerThread, n -> n >= 0, "min-hits-per-search").orElse(-1); } public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { if (numSearchPartitions >= 0) return numSearchPartitions; - return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); + return uniquelyInherited(RankProfile::getNumSearchPartitions, n -> n >= 0, "num-search-partitions").orElse(-1); } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } @@ -761,7 +783,7 @@ public class RankProfile implements Cloneable { public OptionalDouble getTermwiseLimit() { if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); - return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") + return uniquelyInherited(RankProfile::getTermwiseLimit, OptionalDouble::isPresent, "termwise-limit") .orElse(OptionalDouble.empty()); } @@ -769,21 +791,21 @@ public class RankProfile implements Cloneable { if (postFilterThreshold != null) { return OptionalDouble.of(postFilterThreshold); } - return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getPostFilterThreshold, OptionalDouble::isPresent, "post-filter-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getApproximateThreshold() { if (approximateThreshold != null) { return OptionalDouble.of(approximateThreshold); } - return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getApproximateThreshold, OptionalDouble::isPresent, "approximate-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getTargetHitsMaxAdjustmentFactor() { if (targetHitsMaxAdjustmentFactor != null) { return OptionalDouble.of(targetHitsMaxAdjustmentFactor); } - return uniquelyInherited(p -> p.getTargetHitsMaxAdjustmentFactor(), l -> l.isPresent(), "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getTargetHitsMaxAdjustmentFactor, OptionalDouble::isPresent, "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -793,24 +815,34 @@ public class RankProfile implements Cloneable { public Boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); + return uniquelyInherited(RankProfile::getIgnoreDefaultRankFeatures, "ignore-default-rank-features").orElse(false); } public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; - return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); + return uniquelyInherited(RankProfile::getKeepRankCount, c -> c >= 0, "keep-rank-count").orElse(-1); } public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } public double getRankScoreDropLimit() { if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; - return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") + return uniquelyInherited(RankProfile::getRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") .orElse(rankScoreDropLimit); } + public void setSecondPhaseRankScoreDropLimit(double limit) { this.secondPhaseRankScoreDropLimit = limit; } + + public double getSecondPhaseRankScoreDropLimit() { + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + return secondPhaseRankScoreDropLimit; + } + return uniquelyInherited(RankProfile::getSecondPhaseRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "second-phase rank-score-drop-limit") + .orElse(secondPhaseRankScoreDropLimit); + } + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -935,7 +967,7 @@ public class RankProfile implements Cloneable { } private boolean needToUpdateFunctionCache() { - if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; + if (inherited().stream().anyMatch(RankProfile::needToUpdateFunctionCache)) return true; return allFunctionsCached == null; } @@ -943,7 +975,7 @@ public class RankProfile implements Cloneable { /** Returns all filter fields in this profile and any profile it inherits. */ public Set<String> allFilterFields() { - Set<String> inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), + Set<String> inheritedFilterFields = uniquelyInherited(RankProfile::allFilterFields, fields -> ! fields.isEmpty(), "filter fields").orElse(Set.of()); if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); @@ -954,7 +986,7 @@ public class RankProfile implements Cloneable { } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { - if (expression.trim().length() == 0) + if (expression.trim().isEmpty()) throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { @@ -996,7 +1028,8 @@ public class RankProfile implements Cloneable { try { RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); - clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.matchPhase = this.matchPhase; // hmm? + clone.diversity = this.diversity; clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; @@ -1177,7 +1210,7 @@ public class RankProfile implements Cloneable { private Map<Reference, TensorType> featureTypes() { Map<Reference, TensorType> featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), + .collect(Collectors.toMap(Input::name, input -> input.type().tensorType())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); @@ -1494,15 +1527,9 @@ public class RankProfile implements Cloneable { private boolean ascending = false; private int maxHits = 0; // try to get this many hits before degrading the match phase private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. - private DiversitySettings diversity = null; private double evaluationPoint = 0.20; private double prePostFilterTippingPoint = 1.0; - public void setDiversity(DiversitySettings value) { - value.checkValid(); - diversity = value; - } - public void setAscending(boolean value) { ascending = value; } public void setAttribute(String value) { attribute = value; } public void setMaxHits(int value) { maxHits = value; } @@ -1514,7 +1541,6 @@ public class RankProfile implements Cloneable { public String getAttribute() { return attribute; } public int getMaxHits() { return maxHits; } public double getMaxFilterCoverage() { return maxFilterCoverage; } - public DiversitySettings getDiversity() { return diversity; } public double getEvaluationPoint() { return evaluationPoint; } public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } @@ -1678,7 +1704,7 @@ public class RankProfile implements Cloneable { } - public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + public record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { @Override public String toString() { return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; @@ -1699,7 +1725,7 @@ public class RankProfile implements Cloneable { } } - private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + private final List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..127d12594b4 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -721,7 +721,7 @@ public class Schema implements ImmutableSchema { "', but this schema does not exist"); // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later + // and require it to be explicit, so we have the option to support other possibilities later var parentDocument = owner.schemas().get(inherited.get()).getDocument(); if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) throw new IllegalArgumentException(this + " inherits '" + inherited.get() + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index c1b698df55f..3fb185e333d 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -106,15 +106,18 @@ public class IndexInfo extends Derived { } private static boolean isPositionField(ImmutableSDField field) { - return GeoPos.isAnyPos(field); + return (field != null) && GeoPos.isAnyPos(field); + } + private static boolean isMultivalueField(ImmutableSDField field) { + return (field != null) && field.getDataType().isMultivalue(); } @Override protected void derive(ImmutableSDField field, Schema schema) { - derive(field, schema, false); + derive(field, schema, null); } - protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { + protected void derive(ImmutableSDField field, Schema schema, ImmutableSDField parent) { if (field.getDataType().equals(DataType.PREDICATE)) { addIndexCommand(field, CMD_PREDICATE); Index index = field.getIndex(field.getName()); @@ -134,14 +137,13 @@ public class IndexInfo extends Derived { String name = e.getValue(); addIndexAlias(alias, name); } - boolean isPosition = isPositionField(field); if (field.usesStructOrMap()) { for (ImmutableSDField structField : field.getStructFields()) { - derive(structField, schema, isPosition); // Recursion + derive(structField, schema, field); // Recursion } } - if (isPosition) { + if (isPositionField(field)) { addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); } @@ -153,12 +155,12 @@ public class IndexInfo extends Derived { addIndexCommand(field, CMD_LOWERCASE); } - if (field.getDataType().isMultivalue()) { + if (isMultivalueField(field) || isMultivalueField(parent)) { addIndexCommand(field, CMD_MULTIVALUE); } Attribute attribute = field.getAttribute(); - if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { + if ((field.doesAttributing() || (attribute != null && !isPositionField(parent))) && !field.doesIndexing()) { addIndexCommand(field.getName(), CMD_ATTRIBUTE); if (attribute != null && attribute.isFastSearch()) addIndexCommand(field.getName(), CMD_FAST_SEARCH); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index b057624f055..15e5891a3e3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -159,6 +159,7 @@ public class RawRankProfile { private final boolean ignoreDefaultRankFeatures; private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final RankProfile.DiversitySettings diversitySettings; private final int rerankCount; private final int keepRankCount; private final int numThreadsPerSearch; @@ -169,6 +170,7 @@ public class RawRankProfile { private final OptionalDouble approximateThreshold; private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; + private final double secondPhaseRankScoreDropLimit; private final boolean sortBlueprintsByCost; private final boolean alwaysMarkPhraseExpensive; @@ -207,7 +209,8 @@ public class RawRankProfile { rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); - matchPhaseSettings = compiled.getMatchPhaseSettings(); + matchPhaseSettings = compiled.getMatchPhase(); + diversitySettings = compiled.getDiversity(); numThreadsPerSearch = compiled.getNumThreadsPerSearch(); minHitsPerThread = compiled.getMinHitsPerThread(); numSearchPartitions = compiled.getNumSearchPartitions(); @@ -219,6 +222,7 @@ public class RawRankProfile { targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); + secondPhaseRankScoreDropLimit = compiled.getSecondPhaseRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(compiled.getRankProperties()); @@ -486,13 +490,12 @@ public class RawRankProfile { properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); - RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); - if (diversitySettings != null) { - properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); - } + } + if (diversitySettings != null) { + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } if (rerankCount > -1) { properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); @@ -506,6 +509,9 @@ public class RawRankProfile { if (rankScoreDropLimit > -Double.MAX_VALUE) { properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + properties.add(new Pair<>("vespa.hitcollector.secondphase.rankscoredroplimit", secondPhaseRankScoreDropLimit + "")); + } if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index f996b2624db..b91404be2dd 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -183,10 +183,12 @@ public final class SchemaInfo extends Derived { private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (RankProfileInfo rankProfile : rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder() + .name(rankProfile.name()) + .hasSummaryFeatures(rankProfile.hasSummaryFeatures()) + .hasRankFeatures(rankProfile.hasRankFeatures()) + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder() + .useModel(rankProfile.useSignificanceModel())); for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); @@ -226,6 +228,7 @@ public final class SchemaInfo extends Derived { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { @@ -233,11 +236,13 @@ public final class SchemaInfo extends Derived { this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); this.inputs = profile.inputs(); + useSignificanceModel = profile.useSignificanceModel(); } public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } + public boolean useSignificanceModel() { return useSignificanceModel; } public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 398897c6b78..73495c066b9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; -import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 9d68553fa80..33256fa8586 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -33,6 +33,8 @@ public class Matching implements Cloneable, Serializable { private Integer maxLength; /** Maximum number of occurrences for each term */ private Integer maxTermOccurrences; + /** Maximum number of characters in a token. */ + private Integer maxTokenLength; private String exactMatchTerminator = null; @@ -61,6 +63,8 @@ public class Matching implements Cloneable, Serializable { public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public Integer maxTermOccurrences() { return maxTermOccurrences; } public Matching maxTermOccurrences(int maxTermOccurrences) { this.maxTermOccurrences = maxTermOccurrences; return this; } + public Integer maxTokenLength() { return maxTokenLength; } + public Matching maxTokenLength(int maxTokenLength) { this.maxTokenLength = maxTokenLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } public MatchAlgorithm getAlgorithm() { return algorithm; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index f165141b16e..2483fa47667 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -46,7 +46,7 @@ import java.util.TreeMap; * * @author bratseth */ -public class SDField extends Field implements TypedKey, ImmutableSDField { +public class SDField extends Field implements ImmutableSDField { /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ private boolean indexStructureField = false; @@ -315,7 +315,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { supplyStructField.accept(field.getName(), field.getDataType()); } } - if ((subType == null) && (structFields.size() > 0)) { + if ((subType == null) && (!structFields.isEmpty())) { throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + this + " [" + getDataType() + getDataType().getClass() + "] with " + structFields.size() + " struct fields"); @@ -627,7 +627,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); - if (name == null || "".equals(name)) { + if (name == null || name.isEmpty()) { name = getName(); attribute.setName(name); } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java deleted file mode 100644 index 652d21d7f7d..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataType; - -/** - * Common interface for various typed key (or field definitions). - * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping - * - * @author bratseth - */ -public interface TypedKey { - - String getName(); - - void setDataType(DataType type); - - DataType getDataType(); - -} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7659a1e6562..173eebe2a94 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -44,6 +44,7 @@ public class ConvertParsedFields { parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); parsed.getMaxTermOccurrences().ifPresent(maxTermOccurrences -> field.getMatching().maxTermOccurrences(maxTermOccurrences)); + parsed.getMaxTokenLength().ifPresent(maxTokenLength -> field.getMatching().maxTokenLength(maxTokenLength)); parsed.getMatchAlgorithm().ifPresent (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); parsed.getExactTerminator().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 5ccbb7b19a4..ff78a4a3b60 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -38,7 +38,8 @@ public class ConvertParsedRanking { for (String name : parsed.getInherited()) profile.inherit(name); - parsed.isStrict().ifPresent(value -> profile.setStrict(value)); + parsed.isStrict().ifPresent(profile::setStrict); + parsed.isUseSignificanceModel().ifPresent(profile::setUseSignificanceModel); for (var constant : parsed.getConstants().values()) profile.add(constant); @@ -57,39 +58,26 @@ public class ConvertParsedRanking { profile.addFunction(name, parameters, expression, inline); } - parsed.getRankScoreDropLimit().ifPresent - (value -> profile.setRankScoreDropLimit(value)); - parsed.getTermwiseLimit().ifPresent - (value -> profile.setTermwiseLimit(value)); - parsed.getPostFilterThreshold().ifPresent - (value -> profile.setPostFilterThreshold(value)); - parsed.getApproximateThreshold().ifPresent - (value -> profile.setApproximateThreshold(value)); - parsed.getTargetHitsMaxAdjustmentFactor().ifPresent - (value -> profile.setTargetHitsMaxAdjustmentFactor(value)); - parsed.getKeepRankCount().ifPresent - (value -> profile.setKeepRankCount(value)); - parsed.getMinHitsPerThread().ifPresent - (value -> profile.setMinHitsPerThread(value)); - parsed.getNumSearchPartitions().ifPresent - (value -> profile.setNumSearchPartitions(value)); - parsed.getNumThreadsPerSearch().ifPresent - (value -> profile.setNumThreadsPerSearch(value)); - parsed.getReRankCount().ifPresent - (value -> profile.setRerankCount(value)); - - parsed.getMatchPhaseSettings().ifPresent - (value -> profile.setMatchPhaseSettings(value)); - - parsed.getFirstPhaseExpression().ifPresent - (value -> profile.setFirstPhaseRanking(value)); - parsed.getSecondPhaseExpression().ifPresent - (value -> profile.setSecondPhaseRanking(value)); - - parsed.getGlobalPhaseExpression().ifPresent - (value -> profile.setGlobalPhaseRanking(value)); - parsed.getGlobalPhaseRerankCount().ifPresent - (value -> profile.setGlobalPhaseRerankCount(value)); + parsed.getRankScoreDropLimit().ifPresent(profile::setRankScoreDropLimit); + parsed.getSecondPhaseRankScoreDropLimit().ifPresent(profile::setSecondPhaseRankScoreDropLimit); + parsed.getTermwiseLimit().ifPresent(profile::setTermwiseLimit); + parsed.getPostFilterThreshold().ifPresent(profile::setPostFilterThreshold); + parsed.getApproximateThreshold().ifPresent(profile::setApproximateThreshold); + parsed.getTargetHitsMaxAdjustmentFactor().ifPresent(profile::setTargetHitsMaxAdjustmentFactor); + parsed.getKeepRankCount().ifPresent(profile::setKeepRankCount); + parsed.getMinHitsPerThread().ifPresent(profile::setMinHitsPerThread); + parsed.getNumSearchPartitions().ifPresent(profile::setNumSearchPartitions); + parsed.getNumThreadsPerSearch().ifPresent(profile::setNumThreadsPerSearch); + parsed.getReRankCount().ifPresent(profile::setRerankCount); + + parsed.getMatchPhase().ifPresent(profile::setMatchPhase); + parsed.getDiversity().ifPresent(profile::setDiversity); + + parsed.getFirstPhaseExpression().ifPresent(profile::setFirstPhaseRanking); + parsed.getSecondPhaseExpression().ifPresent(profile::setSecondPhaseRanking); + + parsed.getGlobalPhaseExpression().ifPresent(profile::setGlobalPhaseRanking); + parsed.getGlobalPhaseRerankCount().ifPresent(profile::setGlobalPhaseRerankCount); for (var value : parsed.getMatchFeatures()) { profile.addMatchFeatures(value); @@ -101,10 +89,8 @@ public class ConvertParsedRanking { profile.addSummaryFeatures(value); } - parsed.getInheritedMatchFeatures().ifPresent - (value -> profile.setInheritedMatchFeatures(value)); - parsed.getInheritedSummaryFeatures().ifPresent - (value -> profile.setInheritedSummaryFeatures(value)); + parsed.getInheritedMatchFeatures().ifPresent(profile::setInheritedMatchFeatures); + parsed.getInheritedSummaryFeatures().ifPresent(profile::setInheritedSummaryFeatures); if (parsed.getIgnoreDefaultRankFeatures()) { profile.setIgnoreDefaultRankFeatures(true); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 789f7023aed..063962bf0c4 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -133,9 +133,9 @@ public class IntermediateCollection { var parser = new SchemaParser(stream, deployLogger, modelProperties); try { parser.rankProfile(schema); - } catch (ParseException pe) { + } catch (ParseException | TokenMgrException e) { throw new ParseException("Failed parsing rank-profile from '" + reader.getName() + "': " + - stream.formatException(Exceptions.toMessageString(pe))); + stream.formatException(Exceptions.toMessageString(e))); } } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from '" + reader.getName() + "': " + ex.getMessage()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index c7d1a215ce3..bac2c894283 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -23,6 +23,7 @@ public class ParsedMatchSettings { private Integer gramSize = null; private Integer maxLength = null; private Integer maxTermOccurrences = null; + private Integer maxTokenLength = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } @@ -31,6 +32,7 @@ public class ParsedMatchSettings { Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } Optional<Integer> getMaxLength() { return Optional.ofNullable(maxLength); } Optional<Integer> getMaxTermOccurrences() { return Optional.ofNullable(maxTermOccurrences); } + Optional<Integer> getMaxTokenLength() { return Optional.ofNullable(maxTokenLength); } // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } @@ -40,5 +42,6 @@ public class ParsedMatchSettings { void setGramSize(int value) { this.gramSize = value; } void setMaxLength(int value) { this.maxLength = value; } void setMaxTermOccurrences(int value) { this.maxTermOccurrences = value; } + void setMaxTokenLength(int value) { this.maxTokenLength = value; } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index fbbb0c7fe83..2a117a4af4b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.schema.parser; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.DiversitySettings; import com.yahoo.schema.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; @@ -26,6 +27,7 @@ class ParsedRankProfile extends ParsedBlock { private boolean ignoreDefaultRankFeatures = false; private Double rankScoreDropLimit = null; + private Double secondPhaseRankScoreDropLimit = null; private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; @@ -38,12 +40,14 @@ class ParsedRankProfile extends ParsedBlock { private Integer numSearchPartitions = null; private Integer numThreadsPerSearch = null; private Integer reRankCount = null; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; private String firstPhaseExpression = null; private String inheritedSummaryFeatures = null; private String inheritedMatchFeatures = null; private String secondPhaseExpression = null; private Boolean strict = null; + private Boolean useSignificanceModel = null; private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>(); @@ -63,6 +67,7 @@ class ParsedRankProfile extends ParsedBlock { boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } Optional<Double> getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } + Optional<Double> getSecondPhaseRankScoreDropLimit() { return Optional.ofNullable(this.secondPhaseRankScoreDropLimit); } Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } Optional<Double> getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } Optional<Double> getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } @@ -75,7 +80,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<Integer> getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } Optional<Integer> getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } Optional<Integer> getReRankCount() { return Optional.ofNullable(this.reRankCount); } - Optional<MatchPhaseSettings> getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } + Optional<MatchPhaseSettings> getMatchPhase() { return Optional.ofNullable(this.matchPhase); } + Optional<DiversitySettings> getDiversity() { return Optional.ofNullable(this.diversity); } Optional<String> getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } Optional<String> getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } List<ParsedRankFunction> getFunctions() { return List.copyOf(functions.values()); } @@ -96,6 +102,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } Optional<Boolean> isStrict() { return Optional.ofNullable(this.strict); } + Optional<Boolean> isUseSignificanceModel() { return Optional.ofNullable(this.useSignificanceModel); } + void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } @@ -168,9 +176,13 @@ class ParsedRankProfile extends ParsedBlock { this.keepRankCount = count; } - void setMatchPhaseSettings(MatchPhaseSettings settings) { - verifyThat(matchPhaseSettings == null, "already has match-phase"); - this.matchPhaseSettings = settings; + void setMatchPhase(MatchPhaseSettings settings) { + verifyThat(matchPhase == null, "already has match-phase"); + this.matchPhase = settings; + } + void setDiversity(DiversitySettings settings) { + verifyThat(diversity == null, "already has diversity"); + this.diversity = settings; } void setMinHitsPerThread(int minHits) { @@ -193,6 +205,11 @@ class ParsedRankProfile extends ParsedBlock { this.rankScoreDropLimit = limit; } + void setSecondPhaseRankScoreDropLimit(double limit) { + verifyThat(secondPhaseRankScoreDropLimit == null, "already has rank-score-drop-limit for second phase"); + this.secondPhaseRankScoreDropLimit = limit; + } + void setRerankCount(int count) { verifyThat(reRankCount == null, "already has rerank-count"); this.reRankCount = count; @@ -218,6 +235,10 @@ class ParsedRankProfile extends ParsedBlock { this.strict = strict; } + void setUseSignificanceModel(boolean useSignificanceModel) { + verifyThat(this.useSignificanceModel == null, "already has use-model"); + this.useSignificanceModel = useSignificanceModel; + } void setTermwiseLimit(double limit) { verifyThat(termwiseLimit == null, "already has termwise-limit"); this.termwiseLimit = limit; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java index 762279e3871..e66cd62caa8 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java @@ -32,7 +32,7 @@ public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor @Override public void process(boolean validate, boolean documentsOnly) { schema.allImportedFields() - .forEach(field -> setTransform(field)); + .forEach(this::setTransform); } private Stream<SummaryField> getSummaryFieldsForImportedField(ImmutableSDField importedField) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java index 767593b82d0..769f0c9de92 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.TensorDataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.schema.Schema; @@ -45,6 +46,8 @@ public class AttributesImplicitWord extends Processor { private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { // numeric types should not trigger exact-match query parsing if (field.getDataType().getPrimitiveType() instanceof NumericDataType) return false; + // Tensor type should not trigger exact-match query parsing + if (field.getDataType() instanceof TensorDataType) return false; return (! field.hasIndex() && !field.getAttributes().isEmpty() diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java index 5c06ce25184..a8e0f86de8c 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -23,8 +23,8 @@ public class DiversitySettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { - validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + if (rankProfile.getDiversity() != null) { + validate(rankProfile, rankProfile.getDiversity()); } } } @@ -32,6 +32,9 @@ public class DiversitySettingsValidator extends Processor { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); + if ((rankProfile.getMatchPhase() == null) && (rankProfile.getSecondPhaseRanking() == null)) { + throw new IllegalArgumentException("In schema '" + schema.getName() + "', rank-profile '" + rankProfile.name() + "': 'diversity' requires either 'match-phase' or 'second-phase' to be specified."); + } } private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index 056c37a9830..4313ceb4be1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -75,7 +76,11 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + var maxTokenLength = field.getMatching().maxTokenLength(); + if (maxTokenLength == null) { + maxTokenLength = AnnotatorConfig.getDefaultMaxTokenLength(); + } + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, maxTokenLength).convert(field.getIndexingScript())); } } @@ -85,8 +90,12 @@ public class ExactMatch extends Processor { private static class MyProvider extends TypedTransformProvider { - MyProvider(Schema schema) { + private int maxTokenLength; + + MyProvider(Schema schema, int maxTokenLength) + { super(ExactExpression.class, schema); + this.maxTokenLength = maxTokenLength; } @Override @@ -96,7 +105,7 @@ public class ExactMatch extends Processor { @Override protected Expression newTransform(DataType fieldType) { - Expression exp = new ExactExpression(); + Expression exp = new ExactExpression(maxTokenLength); if (fieldType instanceof CollectionDataType) { exp = new ForEachExpression(exp); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java index f3a8f7cee18..d29820e0d51 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -25,7 +25,7 @@ public class MatchPhaseSettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhase(); if (settings != null) { validateMatchPhaseSettings(rankProfile, settings); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index dd36bbb3b61..8b0446364ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -13,6 +13,7 @@ import com.yahoo.schema.document.RankType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.objects.FieldBase; import java.util.Iterator; import java.util.List; @@ -110,8 +111,8 @@ public abstract class Processor { List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { - RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); + for (Iterator<RankProfile.RankSetting> j = profile.declaredRankSettingIterator(); j.hasNext(); ) { + RankProfile.RankSetting setting = j.next(); if (setting.getType().equals(type)) { someRankSettings.add(setting); } @@ -128,7 +129,7 @@ public abstract class Processor { return new IllegalArgumentException(formatError(schemaName, fieldName, msg)); } - protected RuntimeException newProcessException(Schema schema, Field field, String msg) { + protected RuntimeException newProcessException(Schema schema, FieldBase field, String msg) { return newProcessException(schema.getName(), field.getName(), msg); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e29f683761f..3f23cbc9b2d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -64,12 +64,16 @@ public class TextMatch extends Processor { if (fieldMatching != null) { var maxLength = fieldMatching.maxLength(); if (maxLength != null) { - ret.setMaxTokenLength(maxLength); + ret.setMaxTokenizeLength(maxLength); } var maxTermOccurrences = fieldMatching.maxTermOccurrences(); if (maxTermOccurrences != null) { ret.setMaxTermOccurrences(maxTermOccurrences); } + var maxTokenLength = fieldMatching.maxTokenLength(); + if (maxTokenLength != null) { + ret.setMaxTokenLength(maxTokenLength); + } } return ret; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java index 8ccc8870419..8be7be02135 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -29,6 +30,10 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { protected final boolean requiresTransform(Expression exp) { if (exp instanceof OutputExpression) { String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + // Incomplete output expressions never require a transform. + return false; + } if (exp instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) @@ -42,7 +47,7 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { fieldType = field.getDataType(); } else if (exp instanceof SummaryExpression) { - Field field = schema.getSummaryField(fieldName); + SummaryField field = schema.getSummaryField(fieldName); if (field == null) { // Use document field if summary field is not found var sdField = schema.getConcreteField(fieldName); |