diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/derived')
3 files changed, 36 insertions, 25 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 4d6ce783947..52d99c523ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -85,7 +85,8 @@ public class DerivedConfiguration { summaries = new Summaries(search, deployLogger); summaryMap = new SummaryMap(search); juniperrc = new Juniperrc(search); - rankProfileList = new RankProfileList(search, search.rankingConstants(), attributeFields, rankProfileRegistry, queryProfiles, importedModels, deployProperties); + rankProfileList = new RankProfileList(search, search.rankingConstants(), search.rankExpressionFiles(), attributeFields, + rankProfileRegistry, queryProfiles, importedModels, deployProperties); indexingScript = new IndexingScript(search); indexInfo = new IndexInfo(search); indexSchema = new IndexSchema(search); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 42fa1df802b..d414b9ed79f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.searchdefinition.OnnxModel; import com.yahoo.searchdefinition.OnnxModels; +import com.yahoo.searchdefinition.RankExpressionFile; +import com.yahoo.searchdefinition.RankExpressionFiles; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.RankingConstants; @@ -14,6 +16,7 @@ import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.Search; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; import com.yahoo.vespa.model.AbstractService; import java.util.Collection; @@ -25,21 +28,21 @@ import java.util.logging.Logger; * * @author bratseth */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - OnnxModelsConfig.Producer { +public class RankProfileList extends Derived implements RankProfilesConfig.Producer { private static final Logger log = Logger.getLogger(RankProfileList.class.getName()); private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>(); private final RankingConstants rankingConstants; + private final RankExpressionFiles rankExpressionFiles; private final OnnxModels onnxModels; public static RankProfileList empty = new RankProfileList(); private RankProfileList() { - this.rankingConstants = new RankingConstants(); - this.onnxModels = new OnnxModels(); + rankingConstants = new RankingConstants(); + rankExpressionFiles = new RankExpressionFiles(); + onnxModels = new OnnxModels(); } /** @@ -50,6 +53,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ */ public RankProfileList(Search search, RankingConstants rankingConstants, + RankExpressionFiles rankExpressionFiles, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles, @@ -57,7 +61,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ ModelContext.Properties deployProperties) { setName(search == null ? "default" : search.getName()); this.rankingConstants = rankingConstants; - this.onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions + this.rankExpressionFiles = rankExpressionFiles; + onnxModels = search == null ? new OnnxModels() : search.onnxModels(); // as ONNX models come from parsing rank expressions deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, search, attributeFields, deployProperties); } @@ -93,11 +98,9 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ return rankProfiles.get(name); } - public void sendConstantsTo(Collection<? extends AbstractService> services) { + public void sendTo(Collection<? extends AbstractService> services) { rankingConstants.sendTo(services); - } - - public void sendOnnxModelsTo(Collection<? extends AbstractService> services) { + rankExpressionFiles.sendTo(services); onnxModels.sendTo(services); } @@ -111,7 +114,10 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } } - @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { + rankExpressionFiles.asMap().values().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + } + public void getConfig(RankingConstantsConfig.Builder builder) { for (RankingConstant constant : rankingConstants.asMap().values()) { if ("".equals(constant.getFileReference())) @@ -124,7 +130,6 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } } - @Override public void getConfig(OnnxModelsConfig.Builder builder) { for (OnnxModel model : onnxModels.asMap().values()) { if ("".equals(model.getFileReference())) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index c656a426b61..41ac1e17d93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -125,8 +125,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final RankProfile rankProfile; private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); @@ -159,6 +159,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; + private final boolean useExternalExpressionFiles; private Set<String> filterFields = new java.util.LinkedHashSet<>(); @@ -166,11 +167,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { * Creates a raw rank profile from the given rank profile */ Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + AttributeFields attributeFields, ModelContext.Properties deployProperties) { + this.rankProfile = rankProfile; RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); + useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); deriveRankingFeatures(compiled, deployProperties); deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); @@ -230,13 +233,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer { Map<String, String> functionProperties) { SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { + if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; String propertyName = RankingExpression.propertyName(e.getKey()); - if (context.serializedFunctions().containsKey(propertyName)) { - continue; - } + if (context.serializedFunctions().containsKey(propertyName)) continue; + String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); + context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); if (e.getValue().function().returnType().isPresent()) @@ -333,7 +336,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public List<Pair<String, String>> derive() { List<Pair<String, String>> properties = new ArrayList<>(); for (RankProfile.RankProperty property : rankProperties) { - if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { + if (("rankingExpression(" + RankProfile.FIRST_PHASE + ").rankingScript").equals(property.getName())) { // Could have been set by function expansion. Set expressions, then skip this property. try { firstPhaseRanking = new RankingExpression(property.getValue()); @@ -341,7 +344,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { throw new IllegalArgumentException("Could not parse first phase expression", e); } } - else if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) { + else if (("rankingExpression(" + RankProfile.SECOND_PHASE + ").rankingScript").equals(property.getName())) { try { secondPhaseRanking = new RankingExpression(property.getValue()); } catch (ParseException e) { @@ -352,8 +355,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -420,7 +423,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { return properties; } - private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String fileName, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; @@ -428,7 +431,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; - if (expression.getRoot() instanceof ReferenceNode) { + if (useExternalExpressionFiles && (fileName != null)) { + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); + } else if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + name + ")")); |