Merge pull request #28607 from vespa-engine/arnej/declare-more-feature-types

declare types for more rank features

8 files changed, 115 insertions, 11 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java
index 29bd454cc62..5f1cd16c68e 100644
--- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java
@@ -19,6 +19,7 @@ import com.yahoo.searchlib.rankingexpression.Reference;
 import com.yahoo.searchlib.rankingexpression.parser.ParseException;
 import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
+import com.yahoo.tensor.evaluation.TypeContext;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
 import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression;
 
@@ -207,10 +208,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions();
             List<ExpressionFunction> functionExpressions = functions.values().stream().map(RankProfile.RankingExpressionFunction::function).toList();
             Map<String, String> functionProperties = new LinkedHashMap<>();
-            SerializationContext functionSerializationContext = new SerializationContext(functionExpressions,
-                                                                                         Map.of(),
-                                                                                         compiled.typeContext(queryProfiles));
-
+            var typeContext = compiled.typeContext(queryProfiles);
+            SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext);
             if (firstPhaseRanking != null) {
                 functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext));
             }
@@ -220,6 +219,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             if (globalPhaseRanking != null) {
                 functionProperties.putAll(globalPhaseRanking.getRankProperties(functionSerializationContext));
             }
+            deriveFeatureDeclarations(matchFeatures, typeContext, functionProperties);
+            deriveFeatureDeclarations(summaryFeatures, typeContext, functionProperties);
             derivePropertiesAndFeaturesFromFunctions(functions, functionProperties, functionSerializationContext);
             deriveOnnxModelFunctionsAndFeatures(compiled);
 
@@ -228,6 +229,19 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             deriveWeightProperties(compiled);
         }
 
+        private void deriveFeatureDeclarations(Collection<ReferenceNode> features,
+                                               TypeContext typeContext,
+                                               Map<String, String> functionProperties)
+        {
+            for (ReferenceNode feature : features) {
+                var tt = feature.type(typeContext);
+                if (tt != null && tt.rank() > 0) {
+                    String k = "vespa.type.feature." + feature.getName() + feature.getArguments();
+                    functionProperties.put(k, tt.toString());
+                }
+            }
+        }
+
         private void deriveFilterFields(RankProfile rp) {
             filterFields.addAll(rp.allFilterFields());
         }
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg
index b28843736d8..4600884b615 100644
--- a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg
+++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg
@@ -46,6 +46,8 @@ rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript"
 rankprofile[].fef.property[].value "query(yy)"
 rankprofile[].fef.property[].name "rankingExpression(handicap).type"
 rankprofile[].fef.property[].value "tensor(d0[2])"
+rankprofile[].fef.property[].name "vespa.type.feature.attribute(aa)"
+rankprofile[].fef.property[].value "tensor(d1[3])"
 rankprofile[].fef.property[].name "rankingExpression(indirect_a).rankingScript"
 rankprofile[].fef.property[].value "attribute(aa)"
 rankprofile[].fef.property[].name "rankingExpression(indirect_a).type"
diff --git a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg
index 42ddad8abe7..37d84c1a2d9 100644
--- a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg
+++ b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg
@@ -13,6 +13,8 @@ rankprofile[].fef.property[].value "true"
 rankprofile[].fef.property[].name "vespa.type.attribute.tokens"
 rankprofile[].fef.property[].value "tensor(d0[128])"
 rankprofile[].name "using_model"
+rankprofile[].fef.property[].name "vespa.type.feature.attribute(tokens)"
+rankprofile[].fef.property[].value "tensor(d0[128])"
 rankprofile[].fef.property[].name "rankingExpression(__token_length@4d7c1b66085df918).rankingScript"
 rankprofile[].fef.property[].value "reduce(map(query(input), f(x)(x > 0)), sum)"
 rankprofile[].fef.property[].name "rankingExpression(__token_length@a16087c578950aea).rankingScript"
diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
index c5daf67332d..b0f7d0f2477 100644
--- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
@@ -347,6 +347,8 @@ rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpres
 rankprofile[].fef.property[].name "vespa.type.attribute.t1"
 rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].name "withmf"
+rankprofile[].fef.property[].name "vespa.type.feature.mymul"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].fef.property[].name "rankingExpression(myplus).rankingScript"
 rankprofile[].fef.property[].value "attribute(foo1) + attribute(foo2)"
 rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
@@ -405,6 +407,8 @@ rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
 rankprofile[].fef.property[].value "attribute(t1) * query(fromq)"
 rankprofile[].fef.property[].name "rankingExpression(mymul).type"
 rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.feature.attribute(t1)"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "attribute(foo1)"
 rankprofile[].fef.property[].name "vespa.rank.globalphase"
@@ -456,6 +460,8 @@ rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
 rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)"
 rankprofile[].fef.property[].name "rankingExpression(mymul).type"
 rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.feature.mymul"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].fef.property[].name "vespa.rank.firstphase"
 rankprofile[].fef.property[].value "attribute(foo1)"
 rankprofile[].fef.property[].name "vespa.rank.globalphase"
@@ -474,3 +480,43 @@ rankprofile[].fef.property[].name "vespa.type.attribute.t1"
 rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].fef.property[].name "vespa.type.query.v"
 rankprofile[].fef.property[].value "tensor(v[3])"
+rankprofile[].name "withtfl"
+rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
+rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)"
+rankprofile[].fef.property[].name "rankingExpression(mymul).type"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.feature.tensorFromLabels(attribute(labels),m)"
+rankprofile[].fef.property[].value "tensor(m{})"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "attribute(foo1)"
+rankprofile[].fef.property[].name "vespa.rank.globalphase"
+rankprofile[].fef.property[].value "rankingExpression(globalphase)"
+rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(rankingExpression(mymul), sum)"
+rankprofile[].fef.property[].name "vespa.match.feature"
+rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)"
+rankprofile[].fef.property[].name "vespa.type.attribute.t1"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.query.v"
+rankprofile[].fef.property[].value "tensor(v[3])"
+rankprofile[].name "withtfl2"
+rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
+rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)"
+rankprofile[].fef.property[].name "rankingExpression(mymul).type"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.feature.tensorFromLabels(attribute(labels),m)"
+rankprofile[].fef.property[].value "tensor(m{})"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "attribute(foo1)"
+rankprofile[].fef.property[].name "vespa.rank.globalphase"
+rankprofile[].fef.property[].value "rankingExpression(globalphase)"
+rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(rankingExpression(mymul), sum)"
+rankprofile[].fef.property[].name "vespa.match.feature"
+rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)"
+rankprofile[].fef.property[].name "vespa.hidden.matchfeature"
+rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)"
+rankprofile[].fef.property[].name "vespa.type.attribute.t1"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
+rankprofile[].fef.property[].name "vespa.type.query.v"
+rankprofile[].fef.property[].value "tensor(v[3])"
diff --git a/config-model/src/test/derived/rankingexpression/rankexpression.sd b/config-model/src/test/derived/rankingexpression/rankexpression.sd
index 4b3bd516619..b0de2c60299 100644
--- a/config-model/src/test/derived/rankingexpression/rankexpression.sd
+++ b/config-model/src/test/derived/rankingexpression/rankexpression.sd
@@ -408,4 +408,37 @@ schema rankexpression {
       }
   }
 
+  rank-profile withtfl {
+      inputs {
+         query(v) tensor(v[3])
+      }
+      function mymul() {
+        expression: tensorFromLabels(attribute(labels), m) * query(v)
+      }
+      first-phase {
+        expression: attribute(foo1)
+      }
+      global-phase {
+        expression: sum(mymul)
+      }
+      match-features {
+        tensorFromLabels(attribute(labels), m)
+      }
+  }
+
+  rank-profile withtfl2 {
+      inputs {
+         query(v) tensor(v[3])
+      }
+      function mymul() {
+        expression: tensorFromLabels(attribute(labels), m) * query(v)
+      }
+      first-phase {
+        expression: attribute(foo1)
+      }
+      global-phase {
+        expression: sum(mymul)
+      }
+  }
+
 }
diff --git a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg
index ea2d051484b..b6f02133680 100644
--- a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg
+++ b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg
@@ -54,6 +54,8 @@ rankprofile[].fef.property[].value "myplus"
 rankprofile[].fef.property[].name "vespa.type.attribute.t1"
 rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].name "withmf"
+rankprofile[].fef.property[].name "vespa.type.feature.mymul"
+rankprofile[].fef.property[].value "tensor(m{},v[3])"
 rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript"
 rankprofile[].fef.property[].value "attribute(t1) * query(fromq)"
 rankprofile[].fef.property[].name "rankingExpression(mymul).type"
diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java
index b094130ed0f..bd649645a35 100644
--- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java
+++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java
@@ -152,13 +152,15 @@ public class RankingExpressionWithOnnxModelTestCase {
         assertEquals("onnx(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value());
 
         assertEquals("test_summary_features", config.rankprofile(4).name());
-        assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name());
-        assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name());
-        assertEquals("1", config.rankprofile(4).fef().property(3).value());
-        assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name());
-        assertEquals("onnx(another_model).out", config.rankprofile(4).fef().property(4).value());
-        assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name());
-        assertEquals("onnx(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(5).value());
+        assertEquals("vespa.type.feature.onnx(another_model)", config.rankprofile(4).fef().property(0).name());
+        assertEquals("tensor<float>(d0[2])", config.rankprofile(4).fef().property(0).value());
+        assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(2).name());
+        assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(5).name());
+        assertEquals("1", config.rankprofile(4).fef().property(5).value());
+        assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(6).name());
+        assertEquals("onnx(another_model).out", config.rankprofile(4).fef().property(6).value());
+        assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(7).name());
+        assertEquals("onnx(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(7).value());
 
         assertEquals("test_dynamic_model", config.rankprofile(5).name());
         assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name());
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
index 085f80c0ae6..ea8983d039c 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -333,6 +333,9 @@ public class RankProfilesConfigImporter {
         if ( ! expressionMatcher.matches()) return Optional.empty();
         String name = expressionMatcher.group(1);
         String argument = expressionMatcher.group(2);
+        if (name.equals("feature")) {
+            return Optional.of(argument);
+        }
         return Optional.of(name + "(" + argument + ")");
     }