diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-05-12 21:24:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-12 21:24:29 +0200 |
commit | 78e8952ad0ed18c026aaeb7c471dfc886ccde7cb (patch) | |
tree | 9634ca34296bf5cebd52bbdc38450e1e39a77383 /config-model/src/main/java/com | |
parent | 9128b9c93fd58be98409c59892470263f249ec0b (diff) |
Revert "Create distributable constants on deriving"
Diffstat (limited to 'config-model/src/main/java/com')
21 files changed, 263 insertions, 267 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index 69850b21224..812726a609e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.searchdefinition.derived.FileDistributedConstants; import com.yahoo.searchdefinition.document.ImmutableSDField; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; /** @@ -21,8 +21,8 @@ public class DefaultRankProfile extends RankProfile { * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} * to use for storing and looking up rank profiles */ - public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { - super("default", schema, rankProfileRegistry); + public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { + super("default", schema, rankProfileRegistry, rankingConstants); } /** Ignore self inheriting of default as some applications may use that for historical reasons. */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java index be2aa500b2b..11c55521100 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java @@ -14,7 +14,7 @@ public class DistributableResource implements Comparable <DistributableResource> /** The search definition-unique name of this constant */ private final String name; - // TODO: Make path/pathType final + //TODO Make path/pathType final private PathType pathType; private String path; private FileReference fileReference = new FileReference(""); @@ -35,14 +35,14 @@ public class DistributableResource implements Comparable <DistributableResource> this.pathType = type; } - // TODO: Remove and make path/pathType final + //TODO Remove and make path/pathType final public void setFileName(String fileName) { Objects.requireNonNull(fileName, "Filename cannot be null"); this.path = fileName; this.pathType = PathType.FILE; } - // TODO: Remove and make path/pathType final + //TODO Remove and make path/pathType final public void setUri(String uri) { Objects.requireNonNull(uri, "uri cannot be null"); this.path = uri; @@ -65,7 +65,7 @@ public class DistributableResource implements Comparable <DistributableResource> } } - public void register(FileRegistry fileRegistry) { + void register(FileRegistry fileRegistry) { switch (pathType) { case FILE: fileReference = fileRegistry.addFile(path); @@ -91,5 +91,4 @@ public class DistributableResource implements Comparable <DistributableResource> public int compareTo(DistributableResource o) { return name.compareTo(o.getName()); } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java index caaef63fd73..acab2b96772 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java @@ -14,8 +14,9 @@ import java.util.List; */ public class DocumentsOnlyRankProfile extends RankProfile { - public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { - super(name, schema, rankProfileRegistry); + public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, + RankingConstants rankingConstants) { + super(name, schema, rankProfileRegistry, rankingConstants); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java index 1c643292a05..b2f46f4a309 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.SummaryField; import java.io.Reader; @@ -35,7 +34,7 @@ public interface ImmutableSchema { ApplicationPackage applicationPackage(); DeployLogger getDeployLogger(); ModelContext.Properties getDeployProperties(); - Map<Reference, RankProfile.Constant> constants(); + RankingConstants rankingConstants(); LargeRankExpressions rankExpressionFiles(); OnnxModels onnxModels(); Stream<ImmutableSDField> allImportedFields(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java index 90109026dce..6ba17123fb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.FileRegistry; import java.nio.ByteBuffer; +import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -25,7 +26,7 @@ public class RankExpressionBody extends DistributableResource { } } - public void register(FileRegistry fileRegistry) { + void register(FileRegistry fileRegistry) { register(fileRegistry, blob); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 24f6b1390fd..3abf4ec3596 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -130,6 +130,7 @@ public class RankProfile implements Cloneable { /** Global onnx models not tied to a schema */ private final OnnxModels onnxModels; + private final RankingConstants rankingConstants; private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -141,10 +142,11 @@ public class RankProfile implements Cloneable { * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing * and looking up rank profiles. */ - public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { + public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.schema = Objects.requireNonNull(schema, "schema cannot be null"); this.onnxModels = null; + this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; this.applicationPackage = schema.applicationPackage(); this.deployLogger = schema.getDeployLogger(); @@ -156,10 +158,11 @@ public class RankProfile implements Cloneable { * @param name the name of the new profile */ public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, OnnxModels onnxModels) { + RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.schema = null; this.rankProfileRegistry = rankProfileRegistry; + this.rankingConstants = rankingConstants; this.onnxModels = onnxModels; this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; @@ -175,6 +178,11 @@ public class RankProfile implements Cloneable { return applicationPackage; } + /** Returns the ranking constants of the owner of this */ + public RankingConstants rankingConstants() { + return rankingConstants; + } + public Map<String, OnnxModel> onnxModels() { return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); } @@ -407,25 +415,24 @@ public class RankProfile implements Cloneable { return finalSettings; } - public void add(Constant constant) { - constants.put(constant.name(), constant); + public void addConstant(Reference name, Constant value) { + constants.put(name, value); } /** Returns an unmodifiable view of the constants available in this */ public Map<Reference, Constant> getConstants() { + if (inherited().isEmpty()) return new HashMap<>(constants); + Map<Reference, Constant> allConstants = new HashMap<>(); for (var inheritedProfile : inherited()) { - for (var constant : inheritedProfile.getConstants().values()) { - if (allConstants.containsKey(constant.name())) - throw new IllegalArgumentException(constant + "' is present in " + + for (var constant : inheritedProfile.getConstants().entrySet()) { + if (allConstants.containsKey(constant.getKey())) + throw new IllegalArgumentException("Constant '" + constant.getKey() + "' is present in " + inheritedProfile + " inherited by " + this + ", but is also present in another profile inherited by it"); - allConstants.put(constant.name(), constant); + allConstants.put(constant.getKey(), constant.getValue()); } } - - if (schema != null) - allConstants.putAll(schema.constants()); allConstants.putAll(constants); return allConstants; } @@ -1039,7 +1046,9 @@ public class RankProfile implements Cloneable { Map<Reference, TensorType> featureTypes) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); + // Add small and large constants, respectively getConstants().forEach((k, v) -> context.setType(k, v.type())); + rankingConstants().asMap().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); // Add query features from all rank profile types for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { @@ -1418,32 +1427,23 @@ public class RankProfile implements Cloneable { private final Optional<Tensor> value; private final Optional<String> valuePath; - // Always set only if valuePath is set - private final Optional<DistributableResource.PathType> pathType; - public Constant(Reference name, Tensor value) { - this(name, value.type(), Optional.of(value), Optional.empty(), Optional.empty()); + this(name, value.type(), Optional.of(value), Optional.empty()); } public Constant(Reference name, TensorType type, String valuePath) { - this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(DistributableResource.PathType.FILE)); - } - - public Constant(Reference name, TensorType type, String valuePath, DistributableResource.PathType pathType) { - this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(pathType)); + this(name, type, Optional.empty(), Optional.of(valuePath)); } - private Constant(Reference name, TensorType type, Optional<Tensor> value, - Optional<String> valuePath, Optional<DistributableResource.PathType> pathType) { - this.name = Objects.requireNonNull(name); - this.type = Objects.requireNonNull(type); - this.value = Objects.requireNonNull(value); - this.valuePath = Objects.requireNonNull(valuePath); - this.pathType = Objects.requireNonNull(pathType); - + private Constant(Reference name, TensorType type, Optional<Tensor> value, Optional<String> valuePath) { if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + ": Dense tensor dimensions must have a size"); + + this.name = name; + this.type = type; + this.value = value; + this.valuePath = valuePath; } public Reference name() { return name; } @@ -1455,9 +1455,6 @@ public class RankProfile implements Cloneable { /** Returns the path to the value of this, if its value is empty. */ public Optional<String> valuePath() { return valuePath; } - /** Returns the path type, if valuePath is set. */ - public Optional<DistributableResource.PathType> pathType() { return pathType; } - @Override public boolean equals(Object o) { if (o == this) return true; @@ -1467,19 +1464,18 @@ public class RankProfile implements Cloneable { if ( ! other.type().equals(this.type())) return false; if ( ! other.value().equals(this.value())) return false; if ( ! other.valuePath().equals(this.valuePath())) return false; - if ( ! other.pathType().equals(this.pathType())) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, type, value, valuePath, pathType); + return Objects.hash(name, type, value, valuePath); } @Override public String toString() { - return "constant '" + name + "' " + type + ":" + - (value().isPresent() ? value.get().toAbbreviatedString() : " file:" + valuePath.get()); + return "constant '" + name + "' " + type + + (value().isPresent() ? ":" + value.get().toAbbreviatedString() : "file:" + valuePath.get()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 2e815b7b503..bb2dc04cd2a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -31,8 +31,8 @@ public class RankProfileRegistry { public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); return rankProfileRegistry; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java new file mode 100644 index 00000000000..7a2765de852 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.tensor.TensorType; + +/** + * A global ranking constant distributed using file distribution. + * Ranking constants must be sent to some services to be useful - this is done + * by calling the sentTo method during the prepare phase of building models. + * + * @author arnej + * @author bratseth + */ +public class RankingConstant extends DistributableResource { + + private TensorType tensorType = null; + + public RankingConstant(String name) { + super(name); + } + + public RankingConstant(String name, TensorType type, String fileName) { + this(name, type, fileName, PathType.FILE); + } + public RankingConstant(String name, TensorType type, String fileName, PathType pathType) { + super(name, fileName, pathType); + this.tensorType = type; + validate(); + } + + public void setType(TensorType type) { + this.tensorType = type; + } + + public TensorType getTensorType() { return tensorType; } + public String getType() { return tensorType.toString(); } + + public void validate() { + super.validate(); + if (tensorType == null) + throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); + if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + + ": Dense tensor dimensions must have a size"); + } + + @Override + public String toString() { + return super.toString() + "' of type '" + tensorType + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java new file mode 100644 index 00000000000..e10da716335 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.FileRegistry; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +/** + * Constant values for ranking/model execution tied to a schema, or globally to an application package + * + * @author bratseth + */ +public class RankingConstants { + + private final FileRegistry fileRegistry; + + /** The schema this belongs to, or empty if it is global */ + private final Optional<Schema> owner; + + private final Map<String, RankingConstant> constants = new LinkedHashMap<>(); + + private final Object mutex = new Object(); + + public RankingConstants(FileRegistry fileRegistry, Optional<Schema> owner) { + this.fileRegistry = fileRegistry; + this.owner = owner; + } + + public void add(RankingConstant constant) { + synchronized (mutex) { + constant.validate(); + constant.register(fileRegistry); + String name = constant.getName(); + RankingConstant prev = constants.putIfAbsent(name, constant); + if (prev != null) + throw new IllegalArgumentException("Constant '" + name + "' defined twice"); + } + } + + public void putIfAbsent(RankingConstant constant) { + synchronized (mutex) { + constant.validate(); + constant.register(fileRegistry); + String name = constant.getName(); + constants.putIfAbsent(name, constant); + } + } + + public void computeIfAbsent(String name, Function<? super String, ? extends RankingConstant> createConstant) { + synchronized (mutex) { + constants.computeIfAbsent(name, key -> { + RankingConstant constant = createConstant.apply(key); + constant.validate(); + constant.register(fileRegistry); + return constant; + }); + } + } + + /** Returns the ranking constant with the given name, or null if not present */ + public RankingConstant get(String name) { + synchronized (mutex) { + var constant = constants.get(name); + if (constant != null) return constant; + if (owner.isPresent() && owner.get().inherited().isPresent()) + return owner.get().inherited().get().rankingConstants().get(name); + return null; + } + } + + /** Returns a read-only map of the ranking constants in this indexed by name */ + public Map<String, RankingConstant> asMap() { + synchronized (mutex) { + // Shortcuts + if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(constants); + if (constants.isEmpty()) return owner.get().inherited().get().rankingConstants().asMap(); + + var allConstants = new LinkedHashMap<>(owner.get().inherited().get().rankingConstants().asMap()); + allConstants.putAll(constants); + return Collections.unmodifiableMap(allConstants); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index db105edc9d4..203a109acd5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -19,7 +19,6 @@ import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.document.TemporaryImportedFields; import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; @@ -86,9 +85,7 @@ public class Schema implements ImmutableSchema { /** External rank expression files of this */ private final LargeRankExpressions largeRankExpressions; - /** Constants that will be available in all rank profiles. */ - // TODO: Remove on Vespa 9: Should always be in a rank profile - private final Map<Reference, RankProfile.Constant> constants = new LinkedHashMap<>(); + private final RankingConstants rankingConstants; private final OnnxModels onnxModels; @@ -150,6 +147,7 @@ public class Schema implements ImmutableSchema { this.properties = properties; this.documentsOnly = documentsOnly; largeRankExpressions = new LargeRankExpressions(fileRegistry); + rankingConstants = new RankingConstants(fileRegistry, Optional.of(this)); onnxModels = new OnnxModels(fileRegistry, Optional.of(this)); } @@ -226,19 +224,8 @@ public class Schema implements ImmutableSchema { @Override public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } - public void add(RankProfile.Constant constant) { - constants.put(constant.name(), constant); - } - @Override - public Map<Reference, RankProfile.Constant> constants() { - if (inherited().isEmpty()) return Collections.unmodifiableMap(constants); - if (constants.isEmpty()) return inherited().get().constants(); - - Map<Reference, RankProfile.Constant> allConstants = new LinkedHashMap<>(inherited().get().constants()); - allConstants.putAll(constants); - return allConstants; - } + public RankingConstants rankingConstants() { return rankingConstants; } @Override public OnnxModels onnxModels() { return onnxModels; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java index 0e4578878df..acf034362ca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; -import com.yahoo.searchdefinition.derived.FileDistributedConstants; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; @@ -12,8 +11,8 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; */ public class UnrankedRankProfile extends RankProfile { - public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { - super("unranked", schema, rankProfileRegistry); + public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { + super("unranked", schema, rankProfileRegistry, rankingConstants); try { RankingExpression exp = new RankingExpression("value(0)"); this.setFirstPhaseRanking(exp); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 31b6d4c3201..8dec0e8339a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -81,7 +81,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); summaryMap = new SummaryMap(schema); juniperrc = new Juniperrc(schema); - rankProfileList = new RankProfileList(schema, schema.constants(), schema.rankExpressionFiles(), + rankProfileList = new RankProfileList(schema, schema.rankingConstants(), schema.rankExpressionFiles(), schema.onnxModels(), attributeFields, deployState); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java deleted file mode 100644 index c9b0ed2f628..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FileDistributedConstants.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.tensor.TensorType; - -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Function; - -/** - * Constant values for ranking/model execution tied to a rank profile, - * to be distributed as files. - * - * @author bratseth - */ -public class FileDistributedConstants { - - private final FileRegistry fileRegistry; - - private final Map<String, DistributableConstant> constants = new LinkedHashMap<>(); - - public FileDistributedConstants(FileRegistry fileRegistry, Collection<RankProfile.Constant> constants) { - this.fileRegistry = fileRegistry; - for (var constant : constants) { - if (constant.valuePath().isPresent()) - add(new DistributableConstant(constant.name().simpleArgument().get(), - constant.type(), - constant.valuePath().get(), - constant.pathType().get())); - } - } - - public void add(DistributableConstant constant) { - constant.validate(); - constant.register(fileRegistry); - String name = constant.getName(); - DistributableConstant prev = constants.putIfAbsent(name, constant); - if ( prev != null ) - throw new IllegalArgumentException("Constant '" + name + "' defined twice"); - } - - public void putIfAbsent(DistributableConstant constant) { - constant.validate(); - constant.register(fileRegistry); - String name = constant.getName(); - constants.putIfAbsent(name, constant); - } - - public void computeIfAbsent(String name, Function<? super String, ? extends DistributableConstant> createConstant) { - constants.computeIfAbsent(name, key -> { - DistributableConstant constant = createConstant.apply(key); - constant.validate(); - constant.register(fileRegistry); - return constant; - }); - } - - /** Returns a read-only map of the constants in this indexed by name. */ - public Map<String, DistributableConstant> asMap() { - return Collections.unmodifiableMap(constants); - } - - public static class DistributableConstant extends DistributableResource { - - private TensorType tensorType; - - public DistributableConstant(String name, TensorType type, String fileName) { - this(name, type, fileName, PathType.FILE); - } - - public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { - super(name, fileName, pathType); - this.tensorType = type; - validate(); - } - - public void setType(TensorType type) { - this.tensorType = type; - } - - public TensorType getTensorType() { return tensorType; } - public String getType() { return tensorType.toString(); } - - public void validate() { - super.validate(); - if (tensorType == null) - throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); - if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + - ": Dense tensor dimensions must have a size"); - } - - @Override - public String toString() { - return super.toString() + "' of type '" + tensorType + "'"; - } - - } - -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 745d9cfeb96..a55c97f4ebb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -9,7 +9,8 @@ import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; import com.yahoo.searchdefinition.LargeRankExpressions; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.Schema; @@ -18,8 +19,6 @@ import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -39,14 +38,14 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private static final Logger log = Logger.getLogger(RankProfileList.class.getName()); private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>(); - private final FileDistributedConstants constants; + private final RankingConstants rankingConstants; private final LargeRankExpressions largeRankExpressions; private final OnnxModels onnxModels; public static final RankProfileList empty = new RankProfileList(); private RankProfileList() { - constants = new FileDistributedConstants(null, List.of()); + rankingConstants = new RankingConstants(null, Optional.empty()); largeRankExpressions = new LargeRankExpressions(null); onnxModels = new OnnxModels(null, Optional.empty()); } @@ -58,43 +57,18 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ * @param attributeFields the attribute fields to create a ranking for */ public RankProfileList(Schema schema, - Map<Reference, RankProfile.Constant> constantsFromSchema, + RankingConstants rankingConstants, LargeRankExpressions largeRankExpressions, OnnxModels onnxModels, AttributeFields attributeFields, DeployState deployState) { setName(schema == null ? "default" : schema.getName()); - this.constants = deriveFileDistributedConstants(schema, constantsFromSchema, deployState); + this.rankingConstants = rankingConstants; this.largeRankExpressions = largeRankExpressions; this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions deriveRankProfiles(schema, attributeFields, deployState); } - private static FileDistributedConstants deriveFileDistributedConstants(Schema schema, - Map<Reference, RankProfile.Constant> constantsFromSchema, - DeployState deployState) { - Map<Reference, RankProfile.Constant> allFileConstants = new HashMap<>(); - addFileConstants(constantsFromSchema.values(), allFileConstants, schema != null ? schema.toString() : "global"); - for (var profile : deployState.rankProfileRegistry().rankProfilesOf(schema)) - addFileConstants(profile.getConstants().values(), allFileConstants, profile.toString()); - return new FileDistributedConstants(deployState.getFileRegistry(), allFileConstants.values()); - } - - private static void addFileConstants(Collection<RankProfile.Constant> source, - Map<Reference, RankProfile.Constant> destination, - String sourceName) { - for (var constant : source) { - if (constant.valuePath().isEmpty()) continue; - var existing = destination.get(constant.name()); - if ( existing != null && ! constant.equals(existing)) - throw new IllegalArgumentException("Duplicate " + constant + " in " + sourceName + - ": Value reference constants must be unique across all rank profiles"); - destination.put(constant.name(), constant); - } - } - - public FileDistributedConstants constants() { return constants; } - private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { return rank.inheritedNames().isEmpty() || rankProfiles.keySet().containsAll(rank.inheritedNames()) || @@ -104,7 +78,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private void deriveRankProfiles(Schema schema, AttributeFields attributeFields, DeployState deployState) { - if (schema != null) { // profiles belonging to a schema have a default profile + if (schema != null) { // profiles belonging to a search have a default profile RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"), largeRankExpressions, deployState.getQueryProfiles().getRegistry(), @@ -132,7 +106,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ready.forEach(rank -> remaining.remove(rank.name())); } } - private void processRankProfiles(List<RankProfile> ready, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, @@ -187,7 +160,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } public void getConfig(RankingConstantsConfig.Builder builder) { - for (var constant : constants.asMap().values()) { + for (RankingConstant constant : rankingConstants.asMap().values()) { if ("".equals(constant.getFileReference())) log.warning("Illegal file reference " + constant); // Let tests pass ... we should find a better way else diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java index f772c5fe903..b04bfca3f76 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java @@ -5,6 +5,8 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; import java.util.List; @@ -30,7 +32,7 @@ public class ConvertParsedRanking { if (name.equals("default")) { return rankProfileRegistry.get(schema, "default"); } - return new RankProfile(name, schema, rankProfileRegistry); + return new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); } void convertRankProfile(Schema schema, ParsedRankProfile parsed) { @@ -40,8 +42,8 @@ public class ConvertParsedRanking { parsed.isStrict().ifPresent(value -> profile.setStrict(value)); - for (var constant : parsed.getConstants().values()) - profile.add(constant); + for (var constant : parsed.getConstants().entrySet()) + profile.addConstant(constant.getKey(), constant.getValue()); for (var input : parsed.getInputs().entrySet()) profile.addInput(input.getKey(), input.getValue()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java index e56245d1332..d3c24fa9924 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; /** * Class converting a collection of schemas from the intermediate format. @@ -207,14 +208,14 @@ public class ConvertParsedSchemas { if (documentsOnly) { return; // skip ranking-only content, not used for document type generation } - for (var constant : parsed.getConstants()) { - schema.add(constant); + for (var rankingConstant : parsed.getRankingConstants()) { + schema.rankingConstants().add(rankingConstant); } for (var onnxModel : parsed.getOnnxModels()) { schema.onnxModels().add(onnxModel); } - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); var rankConverter = new ConvertParsedRanking(rankProfileRegistry); for (var rankProfile : parsed.getRankProfiles()) { rankConverter.convertRankProfile(schema, rankProfile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 0ade3bfd76b..bead0215b7d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -101,9 +101,9 @@ class ParsedRankProfile extends ParsedBlock { this.inheritedSummaryFeatures = other; } - void add(RankProfile.Constant constant) { - verifyThat(! constants.containsKey(constant.name()), "already has constant", constant.name()); - constants.put(constant.name(), constant); + void addConstant(Reference name, RankProfile.Constant value) { + verifyThat(! constants.containsKey(name), "already has constant", name); + constants.put(name, value); } void addInput(Reference name, RankProfile.Input input) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 2bc10554b25..e4f312e98e3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -2,9 +2,8 @@ package com.yahoo.searchdefinition.parser; import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchlib.rankingexpression.Reference; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -17,7 +16,7 @@ import java.util.Optional; * one schema (.sd) file, using simple data structures * as far as possible. * - * Do not put complicated logic here! + * Do not put advanced logic here! * * @author arnej27959 */ @@ -40,7 +39,7 @@ public class ParsedSchema extends ParsedBlock { private Stemming defaultStemming = null; private final List<ImportedField> importedFields = new ArrayList<>(); private final List<OnnxModel> onnxModels = new ArrayList<>(); - private final Map<Reference, RankProfile.Constant> constants = new LinkedHashMap<>(); + private final List<RankingConstant> rankingConstants = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final List<String> inheritedByDocument = new ArrayList<>(); private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap<>(); @@ -71,12 +70,12 @@ public class ParsedSchema extends ParsedBlock { List<ParsedFieldSet> getFieldSets() { return List.copyOf(fieldSets.values()); } List<ParsedIndex> getIndexes() { return List.copyOf(extraIndexes.values()); } List<ParsedStruct> getStructs() { return List.copyOf(extraStructs.values()); } + List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); } List<String> getInherited() { return List.copyOf(inherited); } List<String> getInheritedByDocument() { return List.copyOf(inheritedByDocument); } List<ParsedRankProfile> getRankProfiles() { return List.copyOf(rankProfiles.values()); } List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } List<ParsedSchema> getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } - List<RankProfile.Constant> getConstants() { return List.copyOf(constants.values()); } void addAnnotation(ParsedAnnotation annotation) { String annName = annotation.name(); @@ -133,8 +132,8 @@ public class ParsedSchema extends ParsedBlock { rankProfiles.put(rpName, profile); } - void add(RankProfile.Constant constant) { - constants.put(constant.name(), constant); + void addRankingConstant(RankingConstant constant) { + rankingConstants.add(constant); } void addStruct(ParsedStruct struct) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index f79f3d9c82c..6c04ebbc31e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -35,7 +35,7 @@ import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.derived.FileDistributedConstants; +import com.yahoo.searchdefinition.RankingConstants; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.searchdefinition.document.SDField; @@ -125,6 +125,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** The global rank profiles of this model */ private final RankProfileList rankProfileList; + /** The global ranking constants of this model */ + private final RankingConstants rankingConstants; + /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; @@ -170,15 +173,16 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri version = deployState.getVespaVersion(); wantedNodeVersion = deployState.getWantedNodeVespaVersion(); fileReferencesRepository = new FileReferencesRepository(deployState.getFileRegistry()); + rankingConstants = new RankingConstants(deployState.getFileRegistry(), Optional.empty()); validationOverrides = deployState.validationOverrides(); applicationPackage = deployState.getApplicationPackage(); provisioned = deployState.provisioned(); VespaModelBuilder builder = new VespaDomBuilder(); root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); - createGlobalRankProfiles(deployState, deployState.getFileRegistry()); + createGlobalRankProfiles(deployState, rankingConstants, deployState.getFileRegistry()); rankProfileList = new RankProfileList(null, // null search -> global - Map.of(), + rankingConstants, new LargeRankExpressions(deployState.getFileRegistry()), new OnnxModels(deployState.getFileRegistry(), Optional.empty()), AttributeFields.empty, @@ -251,6 +255,9 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Returns the application package owning this */ public ApplicationPackage applicationPackage() { return applicationPackage; } + /** Returns the global ranking constants of this */ + public RankingConstants rankingConstants() { return rankingConstants; } + /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { return new VespaModel(new NullConfigModelRegistry(), deployState, false); @@ -275,7 +282,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * Creates a rank profile not attached to any search definition, for each imported model in the application package, * and adds it to the given rank profile registry. */ - private void createGlobalRankProfiles(DeployState deployState, FileRegistry fileRegistry) { + private void createGlobalRankProfiles(DeployState deployState, RankingConstants rankingConstants, FileRegistry fileRegistry) { var importedModels = deployState.getImportedModels().all(); DeployLogger deployLogger = deployState.getDeployLogger(); RankProfileRegistry rankProfileRegistry = deployState.rankProfileRegistry(); @@ -285,11 +292,11 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri for (ImportedMlModel model : importedModels) { // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. OnnxModels onnxModels = onnxModelInfoFromSource(model, fileRegistry); - RankProfile profile = new RankProfile(model.name(), applicationPackage, deployLogger, rankProfileRegistry, onnxModels); + RankProfile profile = new RankProfile(model.name(), applicationPackage, deployLogger, rankProfileRegistry, rankingConstants, onnxModels); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { ConvertedModel convertedModel = ConvertedModel.fromSource(applicationPackage, new ModelName(model.name()), - model.name(), profile, queryProfiles.getRegistry(), model); + model.name(), profile, queryProfiles.getRegistry(), model); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); return convertedModel; })); @@ -302,7 +309,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (modelName.contains(".")) continue; // Name space: Not a global profile // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. OnnxModels onnxModels = onnxModelInfoFromStore(modelName, fileRegistry); - RankProfile profile = new RankProfile(modelName, applicationPackage, deployLogger, rankProfileRegistry, onnxModels); + RankProfile profile = new RankProfile(modelName, applicationPackage, deployLogger, rankProfileRegistry, rankingConstants, onnxModels); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { ConvertedModel convertedModel = ConvertedModel.fromStore(applicationPackage, new ModelName(modelName), modelName, profile); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index af2a781a70c..c2f71c78ce3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -6,8 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; @@ -49,11 +48,11 @@ public class RankingConstantsValidator extends Validator { ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); for (Schema schema : deployState.getSchemas()) { - for (RankProfile.Constant rc : schema.constants().values()) { + for (RankingConstant rc : schema.rankingConstants().asMap().values()) { try { validateRankingConstant(rc, applicationPackage); } catch (InvalidConstantTensorException | FileNotFoundException ex) { - exceptionMessageCollector.add(ex, rc.name().toString(), rc.valuePath().get()); + exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName()); } } } @@ -63,21 +62,20 @@ public class RankingConstantsValidator extends Validator { } } - private void validateRankingConstant(RankProfile.Constant rankingConstant, ApplicationPackage application) throws FileNotFoundException { - // Only validate file backed constants: - if (rankingConstant.valuePath().isEmpty()) return; - if (rankingConstant.pathType().get() != DistributableResource.PathType.FILE) return; + private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException { + // TODO: Handle validation of URI soon too. + if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) { + String constantFile = rankingConstant.getFileName(); + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && constantFile.startsWith(FilesApplicationPackage.preprocessed)) { + constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); + } - String constantFile = rankingConstant.valuePath().get(); - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) - && constantFile.startsWith(FilesApplicationPackage.preprocessed)) { - constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); + ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); + new ConstantTensorJsonValidator().validate(constantFile, + rankingConstant.getTensorType(), + tensorApplicationFile.createReader()); } - - ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); - new ConstantTensorJsonValidator().validate(constantFile, - rankingConstant.type(), - tensorApplicationFile.createReader()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 89d85a9b417..7bc6f95aafe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -13,6 +13,7 @@ import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.FeatureNames; import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -265,11 +266,11 @@ public class ConvertedModel { private static Map<String, ExpressionFunction> convertStored(ModelStore store, RankProfile profile) { for (Pair<String, Tensor> constant : store.readSmallConstants()) { var name = FeatureNames.asConstantFeature(constant.getFirst()); - profile.add(new RankProfile.Constant(name, constant.getSecond())); + profile.addConstant(name, new RankProfile.Constant(name, constant.getSecond())); } - for (RankProfile.Constant constant : store.readLargeConstants()) { - profile.add(constant); + for (RankingConstant constant : store.readLargeConstants()) { + profile.rankingConstants().putIfAbsent(constant); } for (Pair<String, RankingExpression> function : store.readFunctions()) { @@ -298,7 +299,7 @@ public class ConvertedModel { Tensor constantValue = Tensor.from(constantValueString); store.writeSmallConstant(constantName, constantValue); Reference name = FeatureNames.asConstantFeature(constantName); - profile.add(new RankProfile.Constant(name, constantValue)); + profile.addConstant(name, new RankProfile.Constant(name, constantValue)); } private static void transformLargeConstant(ModelStore store, @@ -317,11 +318,10 @@ public class ConvertedModel { constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { - var constantReference = FeatureNames.asConstantFeature(constantName); - if ( ! profile.getConstants().containsKey(constantReference)) { - Path constantPath = store.writeLargeConstant(constantName, constantValue); - profile.add(new RankProfile.Constant(constantReference, constantValue.type(), constantPath.toString())); - } + profile.rankingConstants().computeIfAbsent(constantName, name -> { + Path constantPath = store.writeLargeConstant(name, constantValue); + return new RankingConstant(name, constantValue.type(), constantPath.toString()); + }); } } @@ -548,17 +548,15 @@ public class ConvertedModel { } /** - * Reads the information about all the large constants stored in the application package + * Reads the information about all the large (aka ranking) constants stored in the application package * (the constant value itself is replicated with file distribution). */ - List<RankProfile.Constant> readLargeConstants() { + List<RankingConstant> readLargeConstants() { try { - List<RankProfile.Constant> constants = new ArrayList<>(); + List<RankingConstant> constants = new ArrayList<>(); for (ApplicationFile constantFile : application.getFile(modelFiles.largeConstantsInfoPath()).listFiles()) { String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); - constants.add(new RankProfile.Constant(FeatureNames.asConstantFeature(parts[0]), - TensorType.fromSpec(parts[1]), - parts[2])); + constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); } return constants; } |