Require constant() for large constants and fix a type resolving bug (#9769)

* Require constant() for large constants and fix a type resolving bug

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

* Remove noise

7 files changed, 27 insertions, 128 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
index 2f41b172ab6..1e133d0b8f4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java
@@ -34,6 +34,12 @@ public class FeatureNames {
         return name.equals("attribute") || name.equals("constant") || name.equals("query");
     }
 
+    /** Returns true if this is a constant */
+    public static boolean isConstantFeature(Reference reference) {
+        if ( ! isSimpleFeature(reference)) return false;
+        return reference.name().equals("constant");
+    }
+
     /**
      * Returns the single argument of the given feature name, without any quotes,
      * or empty if it is not a valid query, attribute or constant feature name
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index b3853b36aa5..d738929f721 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -756,7 +756,7 @@ public class RankProfile implements Serializable, Cloneable {
             for (FieldDescription field : queryProfileType.declaredFields().values()) {
                 TensorType type = field.getType().asTensorType();
                 Optional<Reference> feature = Reference.simple(field.getName());
-                if ( ! feature.isPresent() || ! feature.get().name().equals("query")) continue;
+                if ( feature.isEmpty() || ! feature.get().name().equals("query")) continue;
 
                 TensorType existingType = context.getType(feature.get());
                 if ( ! Objects.equals(existingType, context.defaultTypeOf(feature.get())))
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 9804b0b6329..a84db895b02 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -8,7 +8,9 @@ import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.io.IOUtils;
 import com.yahoo.io.reader.NamedReader;
+import com.yahoo.search.query.profile.QueryProfile;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileXMLReader;
 import com.yahoo.searchdefinition.derived.SearchOrderer;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.parser.ParseException;
@@ -394,14 +396,13 @@ public class SearchBuilder {
     }
 
     public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException {
-        return createFromDirectory(dir, new RankProfileRegistry(), new QueryProfileRegistry());
+        return createFromDirectory(dir, new RankProfileRegistry());
     }
     public static SearchBuilder createFromDirectory(String dir,
-                                                    RankProfileRegistry rankProfileRegistry,
-                                                    QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException {
+                                                    RankProfileRegistry rankProfileRegistry) throws IOException, ParseException {
         SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir),
                                                   rankProfileRegistry,
-                                                  queryProfileRegistry);
+                                                  createQueryProfileRegistryFromDirectory(dir));
         for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) {
             builder.importFile(i.next());
         }
@@ -409,6 +410,12 @@ public class SearchBuilder {
         return builder;
     }
 
+    private static QueryProfileRegistry createQueryProfileRegistryFromDirectory(String dir) {
+        File queryProfilesDir = new File(dir, "query-profiles");
+        if ( ! queryProfilesDir.exists()) return new QueryProfileRegistry();
+        return new QueryProfileXMLReader().read(queryProfilesDir.toString());
+    }
+
     // TODO: The build methods below just call the create methods above - remove
 
     /**
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java
index caf5f0442eb..6991e2b978b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java
@@ -49,13 +49,13 @@ public class ConstantTensorTransformer extends ExpressionTransformer<RankProfile
     }
 
     private ExpressionNode transformConstantReference(ReferenceNode node, RankProfileTransformContext context) {
-        Reference constantReference = FeatureNames.asConstantFeature(node.getName());
+        Reference constantReference = node.reference();
+        if ( ! FeatureNames.isConstantFeature(constantReference) && constantReference.isIdentifier())
+            constantReference = FeatureNames.asConstantFeature(node.getName());
+
         Value value = context.constants().get(node.getName());
-        if (value == null || value.type().rank() == 0) {
-            if (context.rankProfile().rankingConstants().get(node.getName()) != null) // Large constants: Transform reference but don't add value
-                return new ReferenceNode(constantReference);
-            return node;
-        }
+        if (value == null || value.type().rank() == 0) return node;
+
         TensorValue tensorValue = (TensorValue)value;
         String tensorType = tensorValue.asTensor().type().toString();
         context.rankProperties().put(constantReference.toString() + ".value", tensorValue.toString());
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
index cbabfffb7a1..6fdf448a39b 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
@@ -6,6 +6,7 @@ import com.yahoo.searchlib.rankingexpression.RankingExpression;
 import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer;
 import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
 import com.yahoo.searchlib.rankingexpression.transform.Simplifier;
+import com.yahoo.searchlib.rankingexpression.transform.TensorMaxMinTransformer;
 
 import java.util.List;
 
@@ -30,7 +31,7 @@ public class ExpressionTransforms {
                                  new ConstantTensorTransformer(),
                                  new FunctionInliner(),
                                  new FunctionShadower(),
-                                 new TensorTransformer(),
+                                 new TensorMaxMinTransformer(),
                                  new Simplifier());
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
index 2c0e1eaa56a..630c8644eb1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java
@@ -24,19 +24,17 @@ public class RankProfileTransformContext extends TransformContext {
     private final ImportedMlModels importedModels;
     private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions;
     private final Map<String, String> rankProperties = new HashMap<>();
-    private final MapEvaluationTypeContext types;
 
     public RankProfileTransformContext(RankProfile rankProfile,
                                        QueryProfileRegistry queryProfiles,
                                        ImportedMlModels importedModels,
                                        Map<String, Value> constants,
                                        Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) {
-        super(constants);
+        super(constants, rankProfile.typeContext(queryProfiles));
         this.rankProfile = rankProfile;
         this.queryProfiles = queryProfiles;
         this.importedModels = importedModels;
         this.inlineFunctions = inlineFunctions;
-        this.types = rankProfile.typeContext(queryProfiles);
     }
 
     public RankProfile rankProfile() { return rankProfile; }
@@ -45,10 +43,4 @@ public class RankProfileTransformContext extends TransformContext {
     public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; }
     public Map<String, String> rankProperties() { return rankProperties; }
 
-    /**
-     * Returns the types known in this context. We may have type information for references
-     * for which no value is available
-     */
-    public MapEvaluationTypeContext types() { return types; }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java
deleted file mode 100644
index 5d03c323803..00000000000
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorTransformer.java
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.searchdefinition.expressiontransforms;
-
-import com.yahoo.searchdefinition.RankProfile;
-import com.yahoo.searchdefinition.RankingConstant;
-import com.yahoo.searchdefinition.document.Attribute;
-import com.yahoo.searchlib.rankingexpression.Reference;
-import com.yahoo.searchlib.rankingexpression.evaluation.Context;
-import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
-import com.yahoo.searchlib.rankingexpression.evaluation.MapContext;
-import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
-import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
-import com.yahoo.searchlib.rankingexpression.evaluation.Value;
-import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
-import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
-import com.yahoo.searchlib.rankingexpression.rule.FunctionNode;
-import com.yahoo.searchlib.rankingexpression.rule.NameNode;
-import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
-import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode;
-import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
-import com.yahoo.tensor.Tensor;
-import com.yahoo.tensor.TensorType;
-import com.yahoo.tensor.evaluation.TypeContext;
-import com.yahoo.tensor.functions.Reduce;
-
-import java.util.List;
-import java.util.Optional;
-
-/**
- * Transforms and simplifies tensor expressions.
- *
- * Currently transforms min(tensor,dim) and max(tensor,dim) to
- * reduce(tensor,min/max,dim). This is necessary as the backend does
- * not recognize these forms of min and max.
- *
- * @author lesters
- */
-public class TensorTransformer extends ExpressionTransformer<RankProfileTransformContext> {
-
-    @Override
-    public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) {
-        if (node instanceof CompositeNode) {
-            node = transformChildren((CompositeNode) node, context);
-        }
-        if (node instanceof FunctionNode) {
-            node = transformFunctionNode((FunctionNode) node, context);
-        }
-        return node;
-    }
-
-    private ExpressionNode transformFunctionNode(FunctionNode node, RankProfileTransformContext context) {
-        switch (node.getFunction()) {
-            case min:
-            case max:
-                return transformMaxAndMinFunctionNode(node, context);
-        }
-        return node;
-    }
-
-    /**
-     * Transforms max and min functions if the first
-     * argument returns a tensor type and the second argument is a valid
-     * dimension in the tensor.
-     */
-    private ExpressionNode transformMaxAndMinFunctionNode(FunctionNode node, RankProfileTransformContext context) {
-        if (node.children().size() != 2) {
-            return node;
-        }
-        ExpressionNode arg1 = node.children().get(0);
-        Optional<String> dimension = dimensionName(node.children().get(1));
-        if (dimension.isPresent()) {
-            TensorType type = arg1.type(context.types());
-            if (type.dimension(dimension.get()).isPresent()) {
-                return replaceMaxAndMinFunction(node);
-            }
-        }
-        return node;
-    }
-
-    private Optional<String> dimensionName(ExpressionNode node) {
-        if (node instanceof ReferenceNode) {
-            Reference reference = ((ReferenceNode)node).reference();
-            if (reference.isIdentifier())
-                return Optional.of(reference.name());
-            else
-                return Optional.empty();
-        }
-        else if (node instanceof NameNode) {
-            return Optional.of(((NameNode)node).getValue());
-        }
-        else {
-            return Optional.empty();
-        }
-    }
-
-    private ExpressionNode replaceMaxAndMinFunction(FunctionNode node) {
-        ExpressionNode arg1 = node.children().get(0);
-        ExpressionNode arg2 = node.children().get(1);
-
-        TensorFunctionNode.TensorFunctionExpressionNode expression = TensorFunctionNode.wrapArgument(arg1);
-        Reduce.Aggregator aggregator = Reduce.Aggregator.valueOf(node.getFunction().name());
-        String dimension = ((ReferenceNode) arg2).getName();
-
-        return new TensorFunctionNode(new Reduce(expression, aggregator, dimension));
-    }
-
-}