diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-05-31 10:26:40 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-05-31 10:26:40 +0200 |
commit | 7e1d9a1d25e6d4c424595bb7595446ec11094301 (patch) | |
tree | 5f1d36416b6780d1efafe310a72f33bef5dd14f2 | |
parent | 2be188c0125913687885b9b7e822ce2f1ed2500d (diff) | |
parent | 8b593130baa769a56ff540c81a57da82cc0a0c9b (diff) |
Merge pull request #18032 from vespa-engine/balder/keep-inheritance-information-and-detect-presence-of-external-files
Use inheritance information from the uncompiled rankprofile to sort o…
10 files changed, 169 insertions, 119 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 39ab443554c..af40dc947c8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -370,7 +370,8 @@ public class RankProfile implements Cloneable { */ public RankingExpression getFirstPhaseRanking() { if (firstPhaseRanking != null) return firstPhaseRanking; - if (getInherited() != null) return getInherited().getFirstPhaseRanking(); + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getFirstPhaseRanking(); return null; } @@ -379,15 +380,30 @@ public class RankProfile implements Cloneable { } public String getUniqueExpressionName(String name) { - return getName() + "_" + name; + return getName().replace('-', '_') + "_" + name; + } + public String resolveExpressionName(String name) { + if (externalFileExpressions.contains(name)) { + return getUniqueExpressionName(name); + } + if (functions.get(name) == null) { + RankProfile inherited = getInherited(); + if (inherited != null) { + return inherited.resolveExpressionName(name); + } + } + return name; } public String getFirstPhaseFile() { String name = FIRST_PHASE; if (externalFileExpressions.contains(name)) { return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); } - if ((firstPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getFirstPhaseFile(); + if (firstPhaseRanking == null) { + RankProfile inherited = getInherited(); + if (inherited != null) { + return getInherited().getFirstPhaseFile(); + } } return null; } @@ -397,8 +413,11 @@ public class RankProfile implements Cloneable { if (externalFileExpressions.contains(name)) { return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); } - if ((secondPhaseRanking == null) && (getInherited() != null)) { - return getInherited().getSecondPhaseFile(); + if (secondPhaseRanking == null) { + RankProfile inherited = getInherited(); + if (inherited != null) { + return getInherited().getSecondPhaseFile(); + } } return null; } @@ -407,8 +426,11 @@ public class RankProfile implements Cloneable { if (externalFileExpressions.contains(name)) { return rankExpressionFiles().get(getUniqueExpressionName(name)).getFileName(); } - if (getInherited() != null) { - return getInherited().getExpressionFile(name); + if (functions.get(name) == null) { + RankProfile inherited = getInherited(); + if (inherited != null) { + return inherited.getExpressionFile(name); + } } return null; } @@ -428,7 +450,8 @@ public class RankProfile implements Cloneable { */ public RankingExpression getSecondPhaseRanking() { if (secondPhaseRanking != null) return secondPhaseRanking; - if (getInherited() != null) return getInherited().getSecondPhaseRanking(); + RankProfile inherited = getInherited(); + if (inherited != null) return inherited.getSecondPhaseRanking(); return null; } @@ -746,6 +769,7 @@ public class RankProfile implements Cloneable { clone.functions = new LinkedHashMap<>(this.functions); clone.filterFields = new HashSet<>(this.filterFields); clone.constants = new HashMap<>(this.constants); + clone.externalFileExpressions = new HashSet(this.externalFileExpressions); return clone; } catch (CloneNotSupportedException e) { @@ -781,6 +805,7 @@ public class RankProfile implements Cloneable { secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions + // TODO This merges all functions from inherited profiles too and erases inheritance information. Not good. functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 41ac1e17d93..5602ba67373 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -55,15 +56,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { /** * Creates a raw rank profile from the given rank profile */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, + AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.getName(); - compressedProperties = compress(new Deriver(rankProfile, queryProfiles, importedModels, attributeFields, deployProperties).derive()); + compressedProperties = compress(new Deriver(rankProfile, rankProfile.compile(queryProfiles, importedModels), + attributeFields, deployProperties).derive()); } /** * Only for testing */ - public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields) { + public RawRankProfile(RankProfile rankProfile, QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, AttributeFields attributeFields) { this(rankProfile, queryProfiles, importedModels, attributeFields, new TestProperties()); } @@ -120,61 +124,81 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private static class Deriver { - /** - * The field rank settings of this profile - */ - private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); - - private final RankProfile rankProfile; - private RankingExpression firstPhaseRanking = null; - private RankingExpression secondPhaseRanking = null; - - private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); + // Due to compiled rankprofiles flattening inheritance we need to use the uncompiled + // to sort out what comes from external files and not. + private final RankProfile unCompiledRankProfile; - private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>(); - - private List<RankProfile.RankProperty> rankProperties = new ArrayList<>(); + private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + private final Set<ReferenceNode> summaryFeatures; + private final Set<ReferenceNode> rankFeatures; + private final List<RankProfile.RankProperty> rankProperties; /** * Rank properties for weight settings to make these available to feature executors */ - private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); - - private boolean ignoreDefaultRankFeatures = false; - - private RankProfile.MatchPhaseSettings matchPhaseSettings = null; - - private int rerankCount = -1; - private int keepRankCount = -1; - private int numThreadsPerSearch = -1; - private int minHitsPerThread = -1; - private int numSearchPartitions = -1; - private double termwiseLimit = 1.0; - private double rankScoreDropLimit = -Double.MAX_VALUE; + private final List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); + + private final boolean ignoreDefaultRankFeatures; + private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final int rerankCount; + private final int keepRankCount; + private final int numThreadsPerSearch; + private final int minHitsPerThread; + private final int numSearchPartitions; + private final double termwiseLimit; + private final double rankScoreDropLimit; /** * The rank type definitions used to derive settings for the native rank features */ private final NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); - private final Map<String, String> attributeTypes; private final Map<String, String> queryFeatureTypes; private final boolean useExternalExpressionFiles; + private final Set<String> filterFields = new java.util.LinkedHashSet<>(); - private Set<String> filterFields = new java.util.LinkedHashSet<>(); + private RankingExpression firstPhaseRanking; + private RankingExpression secondPhaseRanking; /** * Creates a raw rank profile from the given rank profile */ - Deriver(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, - AttributeFields attributeFields, ModelContext.Properties deployProperties) + Deriver(RankProfile unCompiledRankProfile, RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.rankProfile = rankProfile; - RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); + this.unCompiledRankProfile = unCompiledRankProfile; attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); useExternalExpressionFiles = deployProperties.featureFlags().useExternalRankExpressions(); - deriveRankingFeatures(compiled, deployProperties); + firstPhaseRanking = compiled.getFirstPhaseRanking(); + secondPhaseRanking = compiled.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures()); + rankFeatures = compiled.getRankFeatures(); + rerankCount = compiled.getRerankCount(); + matchPhaseSettings = compiled.getMatchPhaseSettings(); + numThreadsPerSearch = compiled.getNumThreadsPerSearch(); + minHitsPerThread = compiled.getMinHitsPerThread(); + numSearchPartitions = compiled.getNumSearchPartitions(); + termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); + keepRankCount = compiled.getKeepRankCount(); + rankScoreDropLimit = compiled.getRankScoreDropLimit(); + ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(compiled.getRankProperties()); + + Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); + List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); + Map<String, String> functionProperties = new LinkedHashMap<>(); + SerializationContext functionSerializationContext = new FunctionSerializationContext(unCompiledRankProfile, functionExpressions, functionProperties); + + if (firstPhaseRanking != null) { + functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); + } + if (secondPhaseRanking != null) { + functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext)); + } + + derivePropertiesAndSummaryFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); + deriveOnnxModelFunctionsAndSummaryFeatures(compiled); + deriveRankTypeSetting(compiled, attributeFields); deriveFilterFields(compiled); deriveWeightProperties(compiled); @@ -184,44 +208,35 @@ public class RawRankProfile implements RankProfilesConfig.Producer { filterFields.addAll(rp.allFilterFields()); } - private void deriveRankingFeatures(RankProfile rankProfile, ModelContext.Properties deployProperties) { - firstPhaseRanking = rankProfile.getFirstPhaseRanking(); - secondPhaseRanking = rankProfile.getSecondPhaseRanking(); - summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); - rankFeatures = rankProfile.getRankFeatures(); - rerankCount = rankProfile.getRerankCount(); - matchPhaseSettings = rankProfile.getMatchPhaseSettings(); - numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); - minHitsPerThread = rankProfile.getMinHitsPerThread(); - numSearchPartitions = rankProfile.getNumSearchPartitions(); - termwiseLimit = rankProfile.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - keepRankCount = rankProfile.getKeepRankCount(); - rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); - ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); - rankProperties = new ArrayList<>(rankProfile.getRankProperties()); - derivePropertiesAndSummaryFeaturesFromFunctions(rankProfile.getFunctions()); - deriveOnnxModelFunctionsAndSummaryFeatures(rankProfile); - } - - private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions) { - if (functions.isEmpty()) return; - - List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); + private static class FunctionSerializationContext extends SerializationContext { + private final RankProfile rankProfile; + FunctionSerializationContext(RankProfile rankProfile, Collection<ExpressionFunction> functions, + Map<String, String> serializedFunctions) + { + super(functions, null, serializedFunctions); + this.rankProfile = rankProfile; + } - if (firstPhaseRanking != null) { - functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); + @Override + public String uniqueName(String functionName) { + return rankProfile.resolveExpressionName(functionName); } - if (secondPhaseRanking != null) { - functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); + @Override + public boolean needSerialization(String functionName) { + return functionName.equals(uniqueName(functionName)) && super.needSerialization(functionName); } + } - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); + private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, + Map<String, String> functionProperties, + SerializationContext functionContext) { + if (functions.isEmpty()) return; + + replaceFunctionSummaryFeatures(functionContext); // First phase, second phase and summary features should add all required functions to the context. // However, we need to add any functions not referenced in those anyway for model-evaluation. - deriveFunctionProperties(functions, functionExpressions, functionProperties); + deriveFunctionProperties(functions, functionProperties, functionContext); for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); @@ -229,15 +244,16 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions, - Map<String, String> functionProperties) { - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + Map<String, String> functionProperties, + SerializationContext context) { for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { - if (useExternalExpressionFiles && rankProfile.getExpressionFile(e.getKey()) != null) continue; + if (useExternalExpressionFiles && unCompiledRankProfile.getExpressionFile(e.getKey()) != null) { + continue; + } String propertyName = RankingExpression.propertyName(e.getKey()); if (context.serializedFunctions().containsKey(propertyName)) continue; - String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = e.getValue().function().getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) @@ -259,7 +275,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { ExpressionFunction function = context.getFunction(referenceNode.getName()); if (function != null) { String propertyName = RankingExpression.propertyName(referenceNode.getName()); - String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + String expressionString = function.getBody().getRoot().toString(context).toString(); context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); @@ -355,8 +371,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>(property.getName(), property.getValue())); } } - properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, rankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); - properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, rankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(firstPhaseRanking, unCompiledRankProfile.getFirstPhaseFile(), RankProfile.FIRST_PHASE)); + properties.addAll(deriveRankingPhaseRankProperties(secondPhaseRanking, unCompiledRankProfile.getSecondPhaseFile(), RankProfile.SECOND_PHASE)); for (FieldRankSettings settings : fieldRankSettings.values()) { properties.addAll(settings.deriveRankProperties()); } @@ -408,9 +424,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } - Iterator filterFieldsIterator = filterFields.iterator(); - while (filterFieldsIterator.hasNext()) { - String fieldName = (String) filterFieldsIterator.next(); + for (String fieldName : filterFields) { properties.add(new Pair<>("vespa.isfilterfield." + fieldName, String.valueOf(true))); } for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { @@ -432,7 +446,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { name = phase; if (useExternalExpressionFiles && (fileName != null)) { - properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + rankProfile.getUniqueExpressionName(name) + ")")); + properties.add(new Pair<>("vespa.rank." + phase, "rankingExpression(" + unCompiledRankProfile.getUniqueExpressionName(name) + ")")); } else if (expression.getRoot() instanceof ReferenceNode) { properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); } else { diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index 0ae31a62b4f..d1f6701e7f1 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -248,7 +248,7 @@ rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "40000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "40000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)" rankprofile[].name "macros-refering-macros-inherited" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" @@ -259,7 +259,7 @@ rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rank rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)" rankprofile[].name "macros-refering-macros-inherited2" rankprofile[].fef.property[].name "rankingExpression(m1).rankingScript" rankprofile[].fef.property[].value "700 * fieldMatch(title).completeness" @@ -276,11 +276,9 @@ rankprofile[].fef.property[].name "rankingExpression(m2).rankingScript" rankprofile[].fef.property[].value "rankingExpression(m1) * 67" rankprofile[].fef.property[].name "rankingExpression(m3).rankingScript" rankprofile[].fef.property[].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))" -rankprofile[].fef.property[].name "rankingExpression(m4).rankingScript" -rankprofile[].fef.property[].value "701 * fieldMatch(title).completeness" rankprofile[].fef.property[].name "rankingExpression(m5).rankingScript" -rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))" +rankprofile[].fef.property[].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(macros_refering_macros_m4))" rankprofile[].fef.property[].name "vespa.rank.secondphase" rankprofile[].fef.property[].value "rankingExpression(secondphase)" rankprofile[].fef.property[].name "rankingExpression(secondphase).rankingScript" -rankprofile[].fef.property[].value "3000 * rankingExpression(m2)" +rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpression(macros_refering_macros_m4)" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index d3e0057cfe1..20f9c7a9160 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -276,7 +276,7 @@ search rankexpression { second-phase { expression { - 40000 * m2 + 40000 * m2 * m4 } } @@ -291,14 +291,9 @@ search rankexpression { ) } } - macro m4() { - expression { - 701 * fieldMatch(title).completeness - } - } second-phase { expression { - 3000 * m2 + 3000 * m2 * m4 } } } @@ -324,4 +319,3 @@ search rankexpression { } - diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 9e958dd4d4c..fbb48f8edf6 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -333,6 +333,7 @@ "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()", + "public java.util.Map getRankProperties(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", "public java.util.Map getRankProperties(java.util.List)", "public static java.lang.String propertyName(java.lang.String)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", @@ -1330,6 +1331,7 @@ "public int hashCode()", "public final boolean equals(java.lang.Object)", "public final java.lang.String toString()", + "public final java.lang.StringBuilder toString(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", "public abstract java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public abstract com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public abstract com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" @@ -1581,6 +1583,8 @@ "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withBindings(java.util.Map)", "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withoutBindings()", "public java.util.Map serializedFunctions()", + "public java.lang.String uniqueName(java.lang.String)", + "public boolean needSerialization(java.lang.String)", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 1f27bc8750e..3eb4f16a9dd 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -19,7 +19,6 @@ import java.io.Reader; import java.io.Serializable; import java.io.StringReader; import java.util.Deque; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -251,18 +250,22 @@ public class RankingExpression implements Serializable { /** * Creates the necessary rank properties required to implement this expression. * - * @param functions the expression functions to expand + * @param context context for serialization * @return a list of named rank properties required to implement this expression */ - public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { + public Map<String, String> getRankProperties(SerializationContext context) { Deque<String> path = new LinkedList<>(); - SerializationContext context = new SerializationContext(functions); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); serializedExpressions.put(propertyName(name), serializedRoot); return serializedExpressions; } + @Deprecated + public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { + return getRankProperties(new SerializationContext(functions)); + } + /** * Returns the rank-property name for a given expression name. * diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java index dfcdf1e2662..151b70d763d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java @@ -30,7 +30,10 @@ public abstract class ExpressionNode implements Serializable { @Override public final String toString() { - return toString(new StringBuilder(), new SerializationContext(), null, null).toString(); + return toString(new SerializationContext()).toString(); + } + public final StringBuilder toString(SerializationContext context) { + return toString(new StringBuilder(), context, null, null); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 335d3861d9d..d33c36f0d3b 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -85,10 +85,9 @@ public final class ReferenceNode extends CompositeNode { path.addLast(myPath); String functionName = getName(); - String functionPropertyName = RankingExpression.propertyName(functionName); - boolean alreadySerialized = context.serializedFunctions().containsKey(functionPropertyName); + boolean needSerialization = (getArguments().size() > 0) || context.needSerialization(functionName); - if ( ! alreadySerialized || getArguments().size() > 0) { + if ( needSerialization) { ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); functionName = instance.getName(); @@ -99,7 +98,7 @@ public final class ReferenceNode extends CompositeNode { context.addFunctionTypeSerialization(functionName, function.returnType().get()); } path.removeLast(); - return string.append("rankingExpression(").append(functionName).append(')'); + return string.append("rankingExpression(").append(context.uniqueName(functionName)).append(')'); } // Not resolved in this context: output as-is diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 92889d6607b..c67a6f66d8c 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -3,6 +3,7 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.TensorType; import java.util.Collection; @@ -103,4 +104,12 @@ public class SerializationContext extends FunctionReferenceContext { public Map<String, String> serializedFunctions() { return serializedFunctions; } + public String uniqueName(String functionName) { + return functionName; + } + + public boolean needSerialization(String functionName) { + return ! serializedFunctions().containsKey(RankingExpression.propertyName(functionName)); + } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index ea09de32137..092faa1934e 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.searchlib.rankingexpression.rule.IfNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.tensor.functions.Reduce; import org.junit.Test; @@ -92,7 +93,7 @@ public class RankingExpressionTestCase { RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(functions); + exp.getRankProperties(new SerializationContext(functions)); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[]]", e.getMessage()); } @@ -100,13 +101,13 @@ public class RankingExpressionTestCase { @Test public void testFunctionCycleSerialization() throws ParseException { - List<ExpressionFunction> funnctions = new ArrayList<>(); - funnctions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); - funnctions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); + List<ExpressionFunction> functions = new ArrayList<>(); + functions.add(new ExpressionFunction("foo", null, new RankingExpression("bar"))); + functions.add(new ExpressionFunction("bar", null, new RankingExpression("foo"))); RankingExpression exp = new RankingExpression("foo"); try { - exp.getRankProperties(funnctions); + exp.getRankProperties(new SerializationContext(functions)); } catch (RuntimeException e) { assertEquals("Cycle in ranking expression function: [foo[], bar[]]", e.getMessage()); } @@ -350,7 +351,7 @@ public class RankingExpressionTestCase { try { RankingExpression expression = new RankingExpression(expressionString); // No functions -> expect one rank property - serializedExpression = expression.getRankProperties(Collections.emptyList()).values().iterator().next(); + serializedExpression = expression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); } catch (ParseException e) { @@ -363,7 +364,7 @@ public class RankingExpressionTestCase { RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again String reserializedExpression = - reparsedExpression.getRankProperties(Collections.emptyList()).values().iterator().next(); + reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, reserializedExpression); } catch (ParseException e) { @@ -383,7 +384,7 @@ public class RankingExpressionTestCase { System.out.println("Parsing expression '" + expressionString + "':"); RankingExpression expression = new RankingExpression(expressionString); - Map<String, String> rankProperties = expression.getRankProperties(functions); + Map<String, String> rankProperties = expression.getRankProperties(new SerializationContext(functions)); if (print) { for (String key : rankProperties.keySet()) System.out.println(key + ": " + rankProperties.get(key)); |