Merge branch 'master' into arnej/handle-dynamic-tensor

10 files changed, 176 insertions, 61 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java
index f9b3bc77040..639930041c3 100644
--- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java
@@ -683,6 +683,25 @@ public class RankProfile implements Cloneable {
         addRankProperty(new RankProperty(name, parameter));
     }
 
+    /*
+     * set a rank-property that should be a single-value parameter;
+     * if the same name is used multiple times, that parameter must be identical each time.
+     */
+    public void setRankProperty(String name, String parameter) {
+        var old = rankProperties.get(name);
+        if (old != null) {
+            if (old.size() != 1) {
+                throw new IllegalStateException("setRankProperty used for multi-valued property " + name);
+            }
+            var oldVal = old.get(0).getValue();
+            if (! oldVal.equals(parameter)) {
+                throw new IllegalArgumentException("setRankProperty would change property " + name + " from " + oldVal + " to " + parameter);
+            }
+        } else {
+            addRankProperty(new RankProperty(name, parameter));
+        }
+    }
+
     private void addRankProperty(RankProperty rankProperty) {
         // Just the usual multimap semantics here
         rankProperties.computeIfAbsent(rankProperty.getName(), (String key) -> new ArrayList<>(1)).add(rankProperty);
@@ -1091,7 +1110,7 @@ public class RankProfile implements Cloneable {
                                                                               inlineFunctions);
         RankingExpression expression = expressionTransforms.transform(function.function().getBody(), context);
         for (Map.Entry<String, String> rankProperty : context.rankProperties().entrySet()) {
-            addRankProperty(rankProperty.getKey(), rankProperty.getValue());
+            setRankProperty(rankProperty.getKey(), rankProperty.getValue());
         }
         return function.withExpression(expression);
     }
diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java
index 34e86cbf4a8..70fcf64dff3 100644
--- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java
@@ -100,7 +100,8 @@ public final class Attribute implements Cloneable, Serializable {
         BOOL("bool", "BOOL"),
         PREDICATE("predicate", "PREDICATE"),
         TENSOR("tensor", "TENSOR"),
-        REFERENCE("reference", "REFERENCE");
+        REFERENCE("reference", "REFERENCE"),
+        RAW("raw", "RAW");
 
         private final String myName;  // different from what name() returns.
         private final String exportAttributeTypeName;
@@ -290,7 +291,7 @@ public final class Attribute implements Cloneable, Serializable {
         } else if (fval instanceof ByteFieldValue) {
             return Type.BYTE;
         } else if (fval instanceof Raw) {
-            return Type.BYTE;
+            return Type.RAW;
         } else if (fval instanceof PredicateFieldValue) {
             return Type.PREDICATE;
         } else if (fval instanceof TensorFieldValue) {
@@ -344,6 +345,7 @@ public final class Attribute implements Cloneable, Serializable {
             case PREDICATE -> DataType.PREDICATE;
             case TENSOR -> DataType.getTensor(tensorType.orElseThrow(IllegalStateException::new));
             case REFERENCE-> createReferenceDataType();
+            case RAW -> DataType.RAW;
             default -> throw new IllegalArgumentException("Unknown attribute type " + attributeType);
         };
     }
diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java
index cfc859345ad..890fa5e7a10 100644
--- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java
+++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java
@@ -12,7 +12,7 @@ import com.yahoo.searchlib.rankingexpression.transform.TransformContext;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
 
@@ -27,7 +27,7 @@ public class RankProfileTransformContext extends TransformContext {
     private final QueryProfileRegistry queryProfiles;
     private final ImportedMlModels importedModels;
     private final Map<String, RankProfile.RankingExpressionFunction> inlineFunctions;
-    private final Map<String, String> rankProperties = new HashMap<>();
+    private final Map<String, String> rankProperties = new LinkedHashMap<>();
 
     public RankProfileTransformContext(RankProfile rankProfile,
                                        QueryProfileRegistry queryProfiles,
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
index 8f7e8daeed0..df4b0d0d941 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java
@@ -89,7 +89,7 @@ public class Processing {
                 OnnxModelConfigGenerator::new,
                 OnnxModelTypeResolver::new,
                 RankingExpressionTypeResolver::new,
-                BoolAttributeValidator::new,
+                SingleValueOnlyAttributeValidator::new,
                 PagedAttributeValidator::new,
                 // These should be last:
                 IndexingValidation::new,
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidator.java
index bdb1eed4b10..b2786e6c785 100644
--- a/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java
+++ b/config-model/src/main/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidator.java
@@ -14,9 +14,9 @@ import com.yahoo.vespa.model.container.search.QueryProfiles;
  *
  * @author geirst
  */
-public class BoolAttributeValidator extends Processor {
+public class SingleValueOnlyAttributeValidator extends Processor {
 
-    public BoolAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+    public SingleValueOnlyAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
         super(schema, deployLogger, rankProfileRegistry, queryProfiles);
     }
 
@@ -27,9 +27,10 @@ public class BoolAttributeValidator extends Processor {
             if (attribute == null) {
                 continue;
             }
-            if (attribute.getType().equals(Attribute.Type.BOOL) &&
+            if ((attribute.getType().equals(Attribute.Type.BOOL) ||
+                    attribute.getType().equals(Attribute.Type.RAW)) &&
                     !attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE)) {
-                fail(schema, field, "Only single value bool attribute fields are supported");
+                fail(schema, field, "Only single value " + attribute.getType().getName() + " attribute fields are supported");
             }
         }
     }
diff --git a/config-model/src/test/derived/rankproperties/rank-profiles.cfg b/config-model/src/test/derived/rankproperties/rank-profiles.cfg
index 3ca44288282..b6f51a88197 100644
--- a/config-model/src/test/derived/rankproperties/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankproperties/rank-profiles.cfg
@@ -50,3 +50,49 @@ rankprofile[].fef.property[].name "vespa.fieldweight.description"
 rankprofile[].fef.property[].value "35"
 rankprofile[].fef.property[].name "vespa.fieldweight.tag"
 rankprofile[].fef.property[].value "88"
+rankprofile[].name "withconstants1"
+rankprofile[].fef.property[].name "constant(a).value"
+rankprofile[].fef.property[].value "tensor(x[2]):[1.0, 2.0]"
+rankprofile[].fef.property[].name "constant(a).type"
+rankprofile[].fef.property[].value "tensor(x[2])"
+rankprofile[].fef.property[].name "constant(b).value"
+rankprofile[].fef.property[].value "tensor(y[3]):[3.0, 4.0, 5.0]"
+rankprofile[].fef.property[].name "constant(b).type"
+rankprofile[].fef.property[].value "tensor(y[3])"
+rankprofile[].fef.property[].name "rankingExpression(usea).rankingScript"
+rankprofile[].fef.property[].value "constant(a) * 1.0"
+rankprofile[].fef.property[].name "rankingExpression(usea).type"
+rankprofile[].fef.property[].value "tensor(x[2])"
+rankprofile[].fef.property[].name "rankingExpression(useb).rankingScript"
+rankprofile[].fef.property[].value "constant(a) * 1.0 + constant(b)"
+rankprofile[].fef.property[].name "rankingExpression(useb).type"
+rankprofile[].fef.property[].value "tensor(x[2],y[3])"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(nativeFieldMatch + constant(a) * 1.0 + constant(b), sum)"
+rankprofile[].name "withconstants2"
+rankprofile[].fef.property[].name "constant(a).value"
+rankprofile[].fef.property[].value "tensor(x[2]):[1.0, 2.0]"
+rankprofile[].fef.property[].name "constant(a).type"
+rankprofile[].fef.property[].value "tensor(x[2])"
+rankprofile[].fef.property[].name "constant(b).value"
+rankprofile[].fef.property[].value "tensor(y[3]):[3.0, 4.0, 5.0]"
+rankprofile[].fef.property[].name "constant(b).type"
+rankprofile[].fef.property[].value "tensor(y[3])"
+rankprofile[].fef.property[].name "rankingExpression(usea).rankingScript"
+rankprofile[].fef.property[].value "constant(a) * 1.0"
+rankprofile[].fef.property[].name "rankingExpression(usea).type"
+rankprofile[].fef.property[].value "tensor(x[2])"
+rankprofile[].fef.property[].name "rankingExpression(useb).rankingScript"
+rankprofile[].fef.property[].value "constant(a) * 1.0 + constant(b)"
+rankprofile[].fef.property[].name "rankingExpression(useb).type"
+rankprofile[].fef.property[].value "tensor(x[2],y[3])"
+rankprofile[].fef.property[].name "rankingExpression(usec).rankingScript"
+rankprofile[].fef.property[].value "constant(a) + constant(b)"
+rankprofile[].fef.property[].name "rankingExpression(usec).type"
+rankprofile[].fef.property[].value "tensor(x[2],y[3])"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(nativeFieldMatch + constant(a) + constant(b), sum)"
diff --git a/config-model/src/test/derived/rankproperties/rankproperties.sd b/config-model/src/test/derived/rankproperties/rankproperties.sd
index a2eb987d4ff..a13b3081ca7 100644
--- a/config-model/src/test/derived/rankproperties/rankproperties.sd
+++ b/config-model/src/test/derived/rankproperties/rankproperties.sd
@@ -59,4 +59,28 @@ schema rankproperties {
     }
   }
 
+  rank-profile withconstants1 {
+    constants {
+      constant(a) tensor(x[2]) : [1,2]
+      constant(b) tensor(y[3]) : [3,4,5]
+    }
+    function inline usea() {
+      expression: constant(a)*1.0
+    }
+    function inline useb() {
+      expression: usea + constant(b)
+    }
+    first-phase {
+      expression: sum(nativeFieldMatch + useb)
+    }
+  }
+
+  rank-profile withconstants2 inherits withconstants1 {
+    function inline usec() {
+      expression: constant(a)+constant(b)
+    }
+    first-phase {
+      expression: sum(nativeFieldMatch + usec)
+    }
+  }
 }
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java
deleted file mode 100644
index f19b1f43115..00000000000
--- a/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.schema.processing;
-
-import com.yahoo.schema.parser.ParseException;
-import org.junit.jupiter.api.Test;
-
-import static com.yahoo.schema.ApplicationBuilder.createFromString;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static com.yahoo.config.model.test.TestUtil.joinLines;
-import static org.junit.jupiter.api.Assertions.fail;
-
-/**
- * @author geirst
- */
-public class BoolAttributeValidatorTestCase {
-
-    @Test
-    void array_of_bool_attribute_is_not_supported() throws ParseException {
-        try {
-            createFromString(getSd("field b type array<bool> { indexing: attribute }"));
-            fail("Expected exception");
-        }
-        catch (IllegalArgumentException e) {
-            assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported",
-                    e.getMessage());
-        }
-    }
-
-    @Test
-    void weigtedset_of_bool_attribute_is_not_supported() throws ParseException {
-        try {
-            createFromString(getSd("field b type weightedset<bool> { indexing: attribute }"));
-            fail("Expected exception");
-        }
-        catch (IllegalArgumentException e) {
-            assertEquals("For schema 'test', field 'b': Only single value bool attribute fields are supported",
-                    e.getMessage());
-        }
-    }
-
-    private String getSd(String field) {
-        return joinLines(
-                "schema test {",
-                "  document test {",
-                "    " + field,
-                "  }",
-                "}");
-    }
-
-}
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java
new file mode 100644
index 00000000000..a7f4125a537
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/schema/processing/SingleValueOnlyAttributeValidatorTestCase.java
@@ -0,0 +1,73 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing;
+
+import com.yahoo.schema.parser.ParseException;
+import org.junit.jupiter.api.Test;
+
+import static com.yahoo.schema.ApplicationBuilder.createFromString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * @author geirst
+ */
+public class SingleValueOnlyAttributeValidatorTestCase {
+
+    private static void array_attribute_is_not_supported(String type) throws ParseException {
+        try {
+            createFromString(getSd("field b type array<" + type + "> { indexing: attribute }"));
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("For schema 'test', field 'b': Only single value " + type + " attribute fields are supported",
+                    e.getMessage());
+        }
+    }
+
+    private static void weightedset_attribute_is_not_supported(String type) throws ParseException {
+        try {
+            createFromString(getSd("field b type weightedset<" + type + "> { indexing: attribute }"));
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            if (type.equals("raw")) {
+                assertEquals("weightedset of complex type '[type BUILTIN] {raw}' is not supported",
+                        e.getMessage());
+            } else {
+                assertEquals("For schema 'test', field 'b': Only single value " + type + " attribute fields are supported",
+                        e.getMessage());
+            }
+        }
+    }
+
+    @Test
+    void array_of_bool_attribute_is_not_supported() throws ParseException {
+        array_attribute_is_not_supported("bool");
+    }
+
+    @Test
+    void weightedset_of_bool_attribute_is_not_supported() throws ParseException {
+        weightedset_attribute_is_not_supported("bool");
+    }
+
+    @Test
+    void array_of_raw_attribute_is_not_supported() throws ParseException {
+        array_attribute_is_not_supported("raw");
+    }
+
+    @Test
+    void weightedset_of_raw_attribute_is_not_supported() throws ParseException {
+        weightedset_attribute_is_not_supported("raw");
+    }
+
+    private static String getSd(String field) {
+        return joinLines(
+                "schema test {",
+                "  document test {",
+                "    " + field,
+                "  }",
+                "}");
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java
index 01e80e0f47a..664cfaf4ad7 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java
@@ -25,8 +25,8 @@ public class VespaMlModelTestCase {
     private final Path applicationDir = Path.fromString("src/test/integration/vespa/");
 
     private final String expectedRankConfig =
-            "constant(constant1).type : tensor(x[3])\n" +
             "constant(constant1).value : tensor(x[3]):[0.5, 1.5, 2.5]\n" +
+            "constant(constant1).type : tensor(x[3])\n" +
             "rankingExpression(foo1).rankingScript : reduce(reduce(input1 * input2, sum, name) * constant(constant1), max, x) * 3.0\n" +
             "rankingExpression(foo1).input1.type : tensor(name{},x[3])\n" +
             "rankingExpression(foo1).input2.type : tensor(x[3])\n" +