diff options
19 files changed, 356 insertions, 108 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index c01b009e93b..a55ce0982dd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -101,6 +101,13 @@ public class FeatureNames { return canonicalize("query(\"" + propertyName + "\")"); } + /** Returns true if this is a cpomstant, attribute, or query feature */ + public static boolean isFeature(String feature) { + return FeatureNames.isConstantFeature(feature) || + FeatureNames.isAttributeFeature(feature) || + FeatureNames.isQueryFeature(feature); + } + public static boolean isConstantFeature(String feature) { return feature.startsWith("constant("); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java new file mode 100644 index 00000000000..38a0a23f48a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -0,0 +1,132 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.rule.Arguments; +import com.yahoo.searchlib.rankingexpression.rule.EvaluationTypeContext; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.MapEvaluationContext; +import com.yahoo.tensor.evaluation.TypeContext; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A context which only contains type information. + * This returns empty tensor types (double) for unknown features which are not + * query, attribute or constant features, as we do not have information about which such + * features exist (but we know those that exist are doubles). + * + * @author bratseth + */ +public class MapEvaluationTypeContext extends FunctionReferenceContext implements EvaluationTypeContext { + + private final Map<String, TensorType> featureTypes = new HashMap<>(); + + public MapEvaluationTypeContext(Collection<ExpressionFunction> functions) { + super(functions); + } + + public MapEvaluationTypeContext(Map<String, ExpressionFunction> functions, + Map<String, String> bindings, + Map<String, TensorType> featureTypes) { + super(functions, bindings); + this.featureTypes.putAll(featureTypes); + } + + public void setType(String name, TensorType type) { + featureTypes.put(FeatureNames.canonicalize(name), type); + } + + // TODO: Remove? + @Override + public TensorType getType(String name) { + if (FeatureNames.isFeature(name)) + return featureTypes.get(FeatureNames.canonicalize(name)); + else + return TensorType.empty; // we do not have type information for these. Correct would be either empty or null + } + + @Override + public TensorType getType(String name, Arguments arguments, String output) { + Optional<String> simpleFeature = simpleFeature(name, arguments); // (all simple feature outputs return the same type) + if (simpleFeature.isPresent()) + return featureTypes.get(simpleFeature.get()); + + Optional<ExpressionFunction> function = functionInvocation(name, output); + if (function.isPresent()) + return function.get().getBody().type(this.withBindings(bind(function.get().arguments()))); + + // We do not know what this is - since we do not have complete knowledge abut the match features + // in Java we must assume this is a match feature and return the double type - which is the type of all + // all match features + return TensorType.empty; + } + + /** + * If the arguments makes a simple feature ("attribute(name)", "constant(name)" or "query(name)", + * it is returned. Otherwise empty is returned. + */ + private Optional<String> simpleFeature(String name, Arguments arguments) { + Optional<String> argument = simpleArgument(arguments); + if ( ! argument.isPresent()) return Optional.empty(); + + // The argument may be a "local value" bound to another value, or else it is the "global" argument of the feature + String actualArgument = bindings.getOrDefault(argument.get(), argument.get()); + + String feature = asFeatureString(name, actualArgument); + if (FeatureNames.isFeature(feature)) + return Optional.of(feature); + else + return Optional.empty(); + } + + private String asFeatureString(String name, String argument) { + return name + "(" + argument + ")"; + } + + /** + * If these arguments contains one simple argument string, it is returned. + * Otherwise null is returned. + */ + private Optional<String> simpleArgument(Arguments arguments) { + if (arguments.expressions().size() != 1) return Optional.empty(); + ExpressionNode argument = arguments.expressions().get(0); + + if ( ! (argument instanceof ReferenceNode)) return Optional.empty(); + ReferenceNode refArgument = (ReferenceNode)argument; + + if ( ! refArgument.isBindableName()) return Optional.empty(); + + return Optional.of(refArgument.getName()); + } + + private Optional<ExpressionFunction> functionInvocation(String name, String output) { + if (output != null) return Optional.empty(); + return Optional.ofNullable(functions().get(name)); + } + + /** Binds the given list of formal arguments to their actual values */ + private Map<String, String> bind(List<String> arguments) { + Map<String, String> bindings = new HashMap<>(arguments.size()); + for (String formalArgument : arguments) + bindings.put(formalArgument, null); // TODO + return bindings; + } + + public Map<String, TensorType> featureTypes() { return Collections.unmodifiableMap(featureTypes); } + + @Override + public MapEvaluationTypeContext withBindings(Map<String, String> bindings) { + if (bindings.isEmpty() && this.bindings.isEmpty()) return this; + return new MapEvaluationTypeContext(functions(), bindings, featureTypes); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java deleted file mode 100644 index e0dc7a2f33c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapTypeContext.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * A context which only contains type information. - * This returns empty tensor types (double) for unknown features which are not - * query, attribute or constant features, as we do not have information about which such - * features exist (but we know those that exist are doubles). - * - * @author bratseth - */ -public class MapTypeContext implements TypeContext { - - private final Map<String, TensorType> featureTypes = new HashMap<>(); - - public void setType(String name, TensorType type) { - featureTypes.put(FeatureNames.canonicalize(name), type); - } - - @Override - public TensorType getType(String name) { - if (FeatureNames.isConstantFeature(name) || - FeatureNames.isAttributeFeature(name) || - FeatureNames.isQueryFeature(name)) - return featureTypes.get(FeatureNames.canonicalize(name)); - else - return TensorType.empty; // we do not have type information for these. Correct would be either empty or null - } - - public Map<String, TensorType> bindings() { return Collections.unmodifiableMap(featureTypes); } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index d1a29271014..51c7618acff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -33,6 +33,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Represents a rank profile - a named set of ranking settings @@ -747,7 +748,9 @@ public class RankProfile implements Serializable, Cloneable { * referable from this rank profile. */ public TypeContext typeContext(QueryProfileRegistry queryProfiles) { - MapTypeContext context = new MapTypeContext(); + MapEvaluationTypeContext context = new MapEvaluationTypeContext(getMacros().values().stream() + .map(Macro::asExpressionFunction) + .collect(Collectors.toList())); // Add small constants getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); @@ -955,7 +958,7 @@ public class RankProfile implements Serializable, Cloneable { return inline && formalParams.size() == 0; // only inline no-arg macros; } - public ExpressionFunction toExpressionMacro() { + public ExpressionFunction asExpressionFunction() { return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index ea02f960800..b02362154d9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -188,7 +188,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (macros.isEmpty()) return; Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { - expressionMacros.put(macro.getKey(), macro.getValue().toExpressionMacro()); + expressionMacros.put(macro.getKey(), macro.getValue().asExpressionFunction()); } Map<String, String> macroProperties = new LinkedHashMap<>(); @@ -223,7 +223,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // Is the feature a macro? if (context.getFunction(referenceNode.getName()) != null) { context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(context, null, null)); + referenceNode.toString(context, null, null)); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java index 2b997aa25f2..f16697b5ba6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java @@ -208,6 +208,10 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil throw new IllegalArgumentException("Model refers Placeholder '" + macroName + "' of type " + requiredType + " but this macro is not present in " + profile); + // TODO: We should verify this in the (function reference(s) this is invoked (starting from first/second + // phase and summary features), as it may only resolve correctly given those bindings + // Or, probably better, annotate the macros with type constraints here and verify during general + // type verification TensorType actualType = macro.getRankingExpression().getRoot().type(profile.typeContext(queryProfiles)); if ( actualType == null) throw new IllegalArgumentException("Model refers Placeholder '" + macroName + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java index a7a5ad58430..39974664d82 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeValidator.java @@ -3,11 +3,11 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.MapTypeContext; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Search; import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -40,15 +40,17 @@ public class RankingExpressionTypeValidator extends Processor { private void validate(RankProfile profile) { profile.parseExpressions(); TypeContext context = profile.typeContext(queryProfiles); - for (RankProfile.Macro macro : profile.getMacros().values()) - ensureValid(macro.getRankingExpression(), "macro '" + macro.getName() + "'", context); + profile.getSummaryFeatures().forEach(f -> ensureValid(f, "summary feature " + f, context)); ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context); ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context); } private TensorType ensureValid(RankingExpression expression, String expressionDescription, TypeContext context) { if (expression == null) return null; + return ensureValid(expression.getRoot(), expressionDescription, context); + } + private TensorType ensureValid(ExpressionNode expression, String expressionDescription, TypeContext context) { TensorType type; try { type = expression.type(context); diff --git a/config-model/src/test/derived/rankexpression/rank-profiles.cfg b/config-model/src/test/derived/rankexpression/rank-profiles.cfg index e890b75770b..f5652c31d2a 100644 --- a/config-model/src/test/derived/rankexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankexpression/rank-profiles.cfg @@ -24,7 +24,7 @@ rankprofile[0].fef.property[10].value "4" rankprofile[0].fef.property[11].name "vespa.dump.feature" rankprofile[0].fef.property[11].value "attribute(foo1).out" rankprofile[0].fef.property[12].name "vespa.dump.feature" -rankprofile[0].fef.property[12].value "attribute(bar1.out)" +rankprofile[0].fef.property[12].value "attribute(bar1)" rankprofile[0].fef.property[13].name "vespa.dump.feature" rankprofile[0].fef.property[13].value "attribute(foo2).out" rankprofile[0].fef.property[14].name "vespa.dump.feature" @@ -64,7 +64,7 @@ rankprofile[2].fef.property[2].value "10 + feature(arg1).out.out" rankprofile[2].fef.property[3].name "vespa.summary.feature" rankprofile[2].fef.property[3].value "attribute(foo1).out" rankprofile[2].fef.property[4].name "vespa.summary.feature" -rankprofile[2].fef.property[4].value "attribute(bar1.out)" +rankprofile[2].fef.property[4].value "attribute(bar1)" rankprofile[2].fef.property[5].name "vespa.summary.feature" rankprofile[2].fef.property[5].value "attribute(foo2).out" rankprofile[2].fef.property[6].name "vespa.summary.feature" diff --git a/config-model/src/test/derived/rankexpression/rankexpression.sd b/config-model/src/test/derived/rankexpression/rankexpression.sd index 8ed1f2bab4c..d3e0057cfe1 100644 --- a/config-model/src/test/derived/rankexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankexpression/rankexpression.sd @@ -5,12 +5,10 @@ search rankexpression { field artist type string { indexing: summary | index - # index-to: artist, default } field title type string { indexing: summary | index - # index-to: title, default } field surl type string { @@ -21,6 +19,38 @@ search rankexpression { indexing: summary | attribute } + field foo1 type int { + indexing: attribute + } + + field foo2 type int { + indexing: attribute + } + + field foo3 type int { + indexing: attribute + } + + field foo4 type int { + indexing: attribute + } + + field bar1 type int { + indexing: attribute + } + + field bar2 type int { + indexing: attribute + } + + field bar3 type int { + indexing: attribute + } + + field bar4 type int { + indexing: attribute + } + } rank-profile default { @@ -33,7 +63,7 @@ search rankexpression { expression: if(3>2,4,2) rerank-count: 10 } - rank-features: attribute(foo1).out attribute(bar1.out) + rank-features: attribute(foo1).out attribute(bar1) rank-features { attribute(foo2).out attribute(bar2).out } rank-features { attribute(foo3).out attribute(bar3).out } @@ -65,7 +95,7 @@ search rankexpression { file:rankexpression } } - summary-features: attribute(foo1).out attribute(bar1.out) + summary-features: attribute(foo1).out attribute(bar1) summary-features { attribute(foo2).out attribute(bar2).out } summary-features { attribute(foo3).out attribute(bar3).out } diff --git a/config-model/src/test/derived/rankexpression/summary.cfg b/config-model/src/test/derived/rankexpression/summary.cfg index 00df2e87144..9752a9f55e3 100644 --- a/config-model/src/test/derived/rankexpression/summary.cfg +++ b/config-model/src/test/derived/rankexpression/summary.cfg @@ -15,9 +15,25 @@ classes[0].fields[5].name "summaryfeatures" classes[0].fields[5].type "featuredata" classes[0].fields[6].name "documentid" classes[0].fields[6].type "longstring" -classes[1].id 1787488393 +classes[1].id 1736696699 classes[1].name "attributeprefetch" classes[1].fields[0].name "year" +classes[].fields[].type "integer" +classes[].fields[].name "foo1" +classes[].fields[].type "integer" +classes[].fields[].name "foo2" +classes[].fields[].type "integer" +classes[].fields[].name "foo3" +classes[].fields[].type "integer" +classes[].fields[].name "foo4" +classes[].fields[].type "integer" +classes[].fields[].name "bar1" +classes[].fields[].type "integer" +classes[].fields[].name "bar2" +classes[].fields[].type "integer" +classes[].fields[].name "bar3" +classes[].fields[].type "integer" +classes[].fields[].name "bar4" classes[1].fields[0].type "integer" classes[1].fields[1].name "rankfeatures" classes[1].fields[1].type "featuredata" diff --git a/config-model/src/test/derived/rankexpression/summarymap.cfg b/config-model/src/test/derived/rankexpression/summarymap.cfg index c810f7282ba..21e6cdf346f 100644 --- a/config-model/src/test/derived/rankexpression/summarymap.cfg +++ b/config-model/src/test/derived/rankexpression/summarymap.cfg @@ -7,4 +7,28 @@ override[1].command "rankfeatures" override[1].arguments "" override[2].field "summaryfeatures" override[2].command "summaryfeatures" -override[2].arguments ""
\ No newline at end of file +override[2].arguments "" +override[].field "foo1" +override[].command "attribute" +override[].arguments "foo1" +override[].field "foo2" +override[].command "attribute" +override[].arguments "foo2" +override[].field "foo3" +override[].command "attribute" +override[].arguments "foo3" +override[].field "foo4" +override[].command "attribute" +override[].arguments "foo4" +override[].field "bar1" +override[].command "attribute" +override[].arguments "bar1" +override[].field "bar2" +override[].command "attribute" +override[].arguments "bar2" +override[].field "bar3" +override[].command "attribute" +override[].arguments "bar3" +override[].field "bar4" +override[].command "attribute" +override[].arguments "bar4"
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java index d9712019bc3..054c9220225 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java @@ -85,8 +85,8 @@ public class TensorTransformTestCase extends SearchDefinitionTestCase { "min(attribute(double_field) + attribute(tensor_field_1),x)"); assertTransformedExpression("reduce(attribute(tensor_field_1)*attribute(tensor_field_2),min,x)", "min(attribute(tensor_field_1) * attribute(tensor_field_2),x)"); - assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)" - , "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); + assertTransformedExpression("reduce(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),min,x)", + "min(join(attribute(tensor_field_1),attribute(tensor_field_2),f(x,y)(x*y)),x)"); assertTransformedExpression("min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)", "min(join(tensor_field_1,tensor_field_2,f(x,y)(x*y)),x)"); // because tensor fields are not in attribute(...) assertTransformedExpression("reduce(join(attribute(tensor_field_1),backend_rank_feature,f(x,y)(x*y)),min,x)", diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 2e2858da238..262aba89f27 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,6 +3,7 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.text.Utf8; @@ -11,9 +12,9 @@ import java.security.NoSuchAlgorithmException; import java.util.*; /** - * <p>A function defined by a ranking expression</p> + * A function defined by a ranking expression * - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen * @author bratseth */ public class ExpressionFunction { @@ -23,7 +24,7 @@ public class ExpressionFunction { private final RankingExpression body; /** - * <p>Constructs a new function</p> + * Constructs a new function * * @param name the name of this function * @param arguments its argument names @@ -43,28 +44,27 @@ public class ExpressionFunction { public RankingExpression getBody() { return body; } /** - * <p>Create and return an instance of this function based on the given - * arguments. If function calls are nested, this call might produce - * additional scripts.</p> + * Creates and returns an instance of this function based on the given + * arguments. If function calls are nested, this call may produce + * additional functions. * * @param context the context used to expand this - * @param arguments the arguments to instantiate on. + * @param argumentValues the arguments to instantiate on. * @param path the expansion path leading to this. * @return the script function instance created. */ - public Instance expand(SerializationContext context, List<ExpressionNode> arguments, Deque<String> path) { + public Instance expand(SerializationContext context, List<ExpressionNode> argumentValues, Deque<String> path) { Map<String, String> argumentBindings = new HashMap<>(); - for (int i = 0; i < this.arguments.size() && i < arguments.size(); ++i) { - argumentBindings.put(this.arguments.get(i), arguments.get(i).toString(context, path, null)); + for (int i = 0; i < arguments.size() && i < arguments.size(); ++i) { + argumentBindings.put(arguments.get(i), argumentValues.get(i).toString(context, path, null)); } - return new Instance(toSymbol(argumentBindings), body.getRoot().toString(context.createBinding(argumentBindings), path, null)); + return new Instance(toSymbol(argumentBindings), body.getRoot().toString(context.withBindings(argumentBindings), path, null)); } /** * Returns a symbolic string that represents this function with a given * list of arguments. The arguments are mangled by hashing the string - * representation of the argument expressions, so we might need to revisit - * this if we start seeing collisions. + * representation of the argument expressions. * * @param argumentBindings the bound arguments to include in the symbolic name. * @return the symbolic name for an instance of this function @@ -85,8 +85,8 @@ public class ExpressionFunction { /** - * <p>Returns a more unique hash code than what Java's own {@link - * String#hashCode()} method would produce.</p> + * Returns a more unique hash code than what Java's own {@link + * String#hashCode()} method would produce. * * @param str The string to hash. * @return A 64 bit long hash code. @@ -136,4 +136,5 @@ public class ExpressionFunction { } } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapTypeContext.java index ff2088263d8..d461ae52cbe 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeMapContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/MapTypeContext.java @@ -13,7 +13,7 @@ import java.util.Map; * * @author bratseth */ -public class TypeMapContext implements TypeContext { +public class MapTypeContext implements TypeContext { private final Map<String, TensorType> featureTypes = new HashMap<>(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EvaluationTypeContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EvaluationTypeContext.java new file mode 100644 index 00000000000..3807fbe2207 --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EvaluationTypeContext.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.rankingexpression.rule; + +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.evaluation.TypeContext; + +public interface EvaluationTypeContext extends TypeContext { + + TensorType getType(String name, Arguments arguments, String output); + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java new file mode 100644 index 00000000000..289495f6ce9 --- /dev/null +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchlib.rankingexpression.rule; + +import com.google.common.collect.ImmutableMap; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * The context of a function invocation. + * + * @author bratseth + */ +public class FunctionReferenceContext { + + /** Expression functions indexed by name */ + private final ImmutableMap<String, ExpressionFunction> functions; + + /** Mapping from argument names to the expressions they resolve to */ + public final Map<String, String> bindings = new HashMap<>(); + + /** Create a context for a single serialization task */ + public FunctionReferenceContext() { + this(Collections.emptyList()); + } + + /** Create a context for a single serialization task */ + public FunctionReferenceContext(Collection<ExpressionFunction> functions) { + this(toMap(functions), Collections.emptyMap()); + } + + public FunctionReferenceContext(Collection<ExpressionFunction> functions, Map<String, String> bindings) { + this(toMap(functions), bindings); + } + + /** Create a context for a single serialization task */ + public FunctionReferenceContext(Map<String, ExpressionFunction> functions) { + this(functions.values()); + } + + /** Create a context for a single serialization task */ + public FunctionReferenceContext(Map<String, ExpressionFunction> functions, Map<String, String> bindings) { + this.functions = ImmutableMap.copyOf(functions); + if (bindings != null) + this.bindings.putAll(bindings); + } + + private static ImmutableMap<String, ExpressionFunction> toMap(Collection<ExpressionFunction> list) { + ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); + for (ExpressionFunction function : list) + mapBuilder.put(function.getName(), function); + return mapBuilder.build(); + } + + /** + * Returns a function or null if it isn't defined in this context + */ + public ExpressionFunction getFunction(String name) { return functions.get(name); } + + protected Map<String, ExpressionFunction> functions() { return functions; } + + /** Returns the resolution of an argument, or null if it isn't defined in this context */ + public String getBinding(String name) { return bindings.get(name); } + + /** Returns a new context with the bindings replaced by the given bindings */ + public FunctionReferenceContext withBindings(Map<String, String> bindings) { + return new FunctionReferenceContext(this.functions, bindings); + } + +} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index b9b377dc0ec..ebfef21a815 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -13,11 +13,13 @@ import java.util.Deque; import java.util.List; /** - * A node referring either to a value in the context or to another named ranking expression. + * A node referring either to a value in the context or to a named ranking expression (function aka macro). * * @author simon * @author bratseth */ +// TODO: Using the same node to represent formal function argument, the function itself, and to features is confusing. +// At least the first two should be split into separate classes. public final class ReferenceNode extends CompositeNode { private final String name, output; @@ -71,7 +73,7 @@ public final class ReferenceNode extends CompositeNode { List<ExpressionNode> myArguments = this.arguments.expressions(); String resolvedArgument = context.getBinding(myName); - if (resolvedArgument != null && this.arguments.expressions().size() == 0 && myOutput == null) { + if (resolvedArgument != null && isBindableName()) { // Replace this whole node with the value of the argument value that it maps to myName = resolvedArgument; myArguments = null; @@ -110,15 +112,18 @@ public final class ReferenceNode extends CompositeNode { return ret.toString(); } + /** Returns whether this is a name that can be bound to a value (during argument passing) */ + public boolean isBindableName() { + return this.arguments.expressions().size() == 0 && output == null; + } + @Override public TensorType type(TypeContext context) { - // Ensure base name (excluding output exists, - // but don't support outputs of different tensor types (not used, so no need) - String name = toString(new SerializationContext(), null, false); - TensorType type = context.getType(name); - + String feature = toString(new SerializationContext(), null, false); + TensorType type = context.getType(feature); + // TensorType type = context.getType(name, arguments, output); TODO if (type == null) - throw new IllegalArgumentException("Unknown feature '" + toString() + "'"); + throw new IllegalArgumentException("Unknown feature '" + feature + "'"); return type; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index ba765d07094..796c13a8669 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -16,17 +16,11 @@ import java.util.Map; * * @author bratseth */ -public class SerializationContext { +public class SerializationContext extends FunctionReferenceContext { - /** Expression functions indexed by name */ - private final ImmutableMap<String, ExpressionFunction> functions; - - /** A cache of already serialized expressions indexed by name */ + /** Serialized form of functions indexed by name */ private final Map<String, String> serializedFunctions; - /** Mapping from argument names to the expressions they resolve to */ - public final Map<String, String> bindings = new HashMap<>(); - /** Create a context for a single serialization task */ public SerializationContext() { this(Collections.emptyList()); @@ -77,17 +71,10 @@ public class SerializationContext { */ public SerializationContext(ImmutableMap<String,ExpressionFunction> functions, Map<String, String> bindings, Map<String, String> serializedFunctions) { - this.functions = functions; + super(functions, bindings); this.serializedFunctions = serializedFunctions; - if (bindings != null) - this.bindings.putAll(bindings); } - /** - * Returns a function or null if it isn't defined in this context - */ - public ExpressionFunction getFunction(String name) { return functions.get(name); } - /** Adds the serialization of a function */ public void addFunctionSerialization(String name, String expressionString) { serializedFunctions.put(name, expressionString); @@ -98,17 +85,9 @@ public class SerializationContext { return serializedFunctions.get(name); } - /** - * Returns the resolution of an argument, or null if it isn't defined in this context - */ - public String getBinding(String name) { return bindings.get(name); } - - /** - * Returns a new context which shares the functions and serialized function map with this but has different - * arguments. - */ - public SerializationContext createBinding(Map<String, String> arguments) { - return new SerializationContext(this.functions, arguments, this.serializedFunctions); + @Override + public SerializationContext withBindings(Map<String, String> bindings) { + return new SerializationContext(functions().values(), bindings, this.serializedFunctions); } public Map<String, String> serializedFunctions() { return serializedFunctions; } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java index c882c887c8d..18444c7f8b6 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/TypeResolutionTestCase.java @@ -18,7 +18,7 @@ public class TypeResolutionTestCase { @Test public void testTypeResolution() { - TypeMapContext context = new TypeMapContext(); + MapTypeContext context = new MapTypeContext(); context.setType("query(x1)", TensorType.fromSpec("tensor(x[])")); context.setType("query(x2)", TensorType.fromSpec("tensor(x[10])")); context.setType("query(y1)", TensorType.fromSpec("tensor(y[])")); |