diff options
author | Jon Bratseth <bratseth@oath.com> | 2019-06-13 07:12:23 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-13 07:12:23 +0200 |
commit | 0e20abc33aa88066aedadd43b09353d115d5928b (patch) | |
tree | 159df75408a4eb188b13a532fb42b8896083cfc9 /config-model/src/main/java/com/yahoo | |
parent | 17b6704b20a073a4961baefd1be58dd48012bec4 (diff) |
Revert "Revert "Require constant() for large constants and fix a type resolving bug""
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
7 files changed, 27 insertions, 128 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java index 2f41b172ab6..1e133d0b8f4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java @@ -34,6 +34,12 @@ public class FeatureNames { return name.equals("attribute") || name.equals("constant") || name.equals("query"); } + /** Returns true if this is a constant */ + public static boolean isConstantFeature(Reference reference) { + if ( ! isSimpleFeature(reference)) return false; + return reference.name().equals("constant"); + } + /** * Returns the single argument of the given feature name, without any quotes, * or empty if it is not a valid query, attribute or constant feature name diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index b3853b36aa5..d738929f721 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -756,7 +756,7 @@ public class RankProfile implements Serializable, Cloneable { for (FieldDescription field : queryProfileType.declaredFields().values()) { TensorType type = field.getType().asTensorType(); Optional<Reference> feature = Reference.simple(field.getName()); - if ( ! feature.isPresent() || ! feature.get().name().equals("query")) continue; + if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue; TensorType existingType = context.getType(feature.get()); if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get()))) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 9804b0b6329..a84db895b02 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -8,7 +8,9 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.searchdefinition.derived.SearchOrderer; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; @@ -394,14 +396,13 @@ public class SearchBuilder { } public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { - return createFromDirectory(dir, new RankProfileRegistry(), new QueryProfileRegistry()); + return createFromDirectory(dir, new RankProfileRegistry()); } public static SearchBuilder createFromDirectory(String dir, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry, - queryProfileRegistry); + createQueryProfileRegistryFromDirectory(dir)); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } @@ -409,6 +410,12 @@ public class SearchBuilder { return builder; } + private static QueryProfileRegistry createQueryProfileRegistryFromDirectory(String dir) { + File queryProfilesDir = new File(dir, "query-profiles"); + if ( ! queryProfilesDir.exists()) return new QueryProfileRegistry(); + return new QueryProfileXMLReader().read(queryProfilesDir.toString()); + } + // TODO: The build methods below just call the create methods above - remove /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java index caf5f0442eb..6991e2b978b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java @@ -49,13 +49,13 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile } private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) { - Reference constantReference = FeatureNames.asConstantFeature(node.getName()); + Reference constantReference = node.reference(); + if ( ! FeatureNames.isConstantFeature(constantReference) && constantReference.isIdentifier()) + constantReference = FeatureNames.asConstantFeature(node.getName()); + Value value = context.constants().get(node.getName()); - if (value == null || value.type().rank() == 0) { - if (context.rankProfile().rankingConstants().get(node.getName()) != null) // Large constants: Transform reference but don't add value - return new ReferenceNode(constantReference); - return node; - } + if (value == null || value.type().rank() == 0) return node; + TensorValue tensorValue = (TensorValue)value; String tensorType = tensorValue.asTensor().type().toString(); context.rankProperties().put(constantReference.toString() + ".value", tensorValue.toString()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java index cbabfffb7a1..6fdf448a39b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java @@ -6,6 +6,7 @@ import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; import com.yahoo.searchlib.rankingexpression.transform.Simplifier; +import com.yahoo.searchlib.rankingexpression.transform.TensorMaxMinTransformer; import java.util.List; @@ -30,7 +31,7 @@ public class ExpressionTransforms { new ConstantTensorTransformer(), new FunctionInliner(), new FunctionShadower(), - new TensorTransformer(), + new TensorMaxMinTransformer(), new Simplifier()); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java index 2c0e1eaa56a..630c8644eb1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java @@ -24,19 +24,17 @@ public class RankProfileTransformContext extends TransformContext { private final ImportedMlModels importedModels; private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions; private final Map<String, String> rankProperties = new HashMap<>(); - private final MapEvaluationTypeContext types; public RankProfileTransformContext(RankProfile rankProfile, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, Map<String, Value> constants, Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { - super(constants); + super(constants, rankProfile.typeContext(queryProfiles)); this.rankProfile = rankProfile; this.queryProfiles = queryProfiles; this.importedModels = importedModels; this.inlineFunctions = inlineFunctions; - this.types = rankProfile.typeContext(queryProfiles); } public RankProfile rankProfile() { return rankProfile; } @@ -45,10 +43,4 @@ public class RankProfileTransformContext extends TransformContext { public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } - /** - * Returns the types known in this context. We may have type information for references - * for which no value is available - */ - public MapEvaluationTypeContext types() { return types; } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java deleted file mode 100644 index 5d03c323803..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; - -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.Context; -import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; -import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; -import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; -import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; -import com.yahoo.searchlib.rankingexpression.rule.FunctionNode; -import com.yahoo.searchlib.rankingexpression.rule.NameNode; -import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; -import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; -import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.tensor.evaluation.TypeContext; -import com.yahoo.tensor.functions.Reduce; - -import java.util.List; -import java.util.Optional; - -/** - * Transforms and simplifies tensor expressions. - * - * Currently transforms min(tensor,dim) and max(tensor,dim) to - * reduce(tensor,min/max,dim). This is necessary as the backend does - * not recognize these forms of min and max. - * - * @author lesters - */ -public class TensorTransformer extends ExpressionTransformer<RankProfileTransformContext> { - - @Override - public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) { - if (node instanceof CompositeNode) { - node = transformChildren((CompositeNode) node, context); - } - if (node instanceof FunctionNode) { - node = transformFunctionNode((FunctionNode) node, context); - } - return node; - } - - private ExpressionNode transformFunctionNode(FunctionNode node, RankProfileTransformContext context) { - switch (node.getFunction()) { - case min: - case max: - return transformMaxAndMinFunctionNode(node, context); - } - return node; - } - - /** - * Transforms max and min functions if the first - * argument returns a tensor type and the second argument is a valid - * dimension in the tensor. - */ - private ExpressionNode transformMaxAndMinFunctionNode(FunctionNode node, RankProfileTransformContext context) { - if (node.children().size() != 2) { - return node; - } - ExpressionNode arg1 = node.children().get(0); - Optional<String> dimension = dimensionName(node.children().get(1)); - if (dimension.isPresent()) { - TensorType type = arg1.type(context.types()); - if (type.dimension(dimension.get()).isPresent()) { - return replaceMaxAndMinFunction(node); - } - } - return node; - } - - private Optional<String> dimensionName(ExpressionNode node) { - if (node instanceof ReferenceNode) { - Reference reference = ((ReferenceNode)node).reference(); - if (reference.isIdentifier()) - return Optional.of(reference.name()); - else - return Optional.empty(); - } - else if (node instanceof NameNode) { - return Optional.of(((NameNode)node).getValue()); - } - else { - return Optional.empty(); - } - } - - private ExpressionNode replaceMaxAndMinFunction(FunctionNode node) { - ExpressionNode arg1 = node.children().get(0); - ExpressionNode arg2 = node.children().get(1); - - TensorFunctionNode.TensorFunctionExpressionNode expression = TensorFunctionNode.wrapArgument(arg1); - Reduce.Aggregator aggregator = Reduce.Aggregator.valueOf(node.getFunction().name()); - String dimension = ((ReferenceNode) arg2).getName(); - - return new TensorFunctionNode(new Reduce(expression, aggregator, dimension)); - } - -} |