diff options
author | Arne H Juul <arnej27959@users.noreply.github.com> | 2023-09-22 15:06:25 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-22 15:06:25 +0200 |
commit | 396b51174560f3f8fa422a07945603358f328f9b (patch) | |
tree | 0f90c9f49164f2a99c5fed6202381a2ce13075ac | |
parent | a7431d64d5983fb6b45485a057cc1c71fe86de4e (diff) | |
parent | 24d0dcef6280b5aa059d23f87c232ef6c6ac8502 (diff) |
Merge pull request #28607 from vespa-engine/arnej/declare-more-feature-types
declare types for more rank features
8 files changed, 115 insertions, 11 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 29bd454cc62..5f1cd16c68e 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -19,6 +19,7 @@ import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; +import com.yahoo.tensor.evaluation.TypeContext; import com.yahoo.vespa.config.search.RankProfilesConfig; import static com.yahoo.searchlib.rankingexpression.Reference.wrapInRankingExpression; @@ -207,10 +208,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); List<ExpressionFunction> functionExpressions = functions.values().stream().map(RankProfile.RankingExpressionFunction::function).toList(); Map<String, String> functionProperties = new LinkedHashMap<>(); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, - Map.of(), - compiled.typeContext(queryProfiles)); - + var typeContext = compiled.typeContext(queryProfiles); + SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); } @@ -220,6 +219,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (globalPhaseRanking != null) { functionProperties.putAll(globalPhaseRanking.getRankProperties(functionSerializationContext)); } + deriveFeatureDeclarations(matchFeatures, typeContext, functionProperties); + deriveFeatureDeclarations(summaryFeatures, typeContext, functionProperties); derivePropertiesAndFeaturesFromFunctions(functions, functionProperties, functionSerializationContext); deriveOnnxModelFunctionsAndFeatures(compiled); @@ -228,6 +229,19 @@ public class RawRankProfile implements RankProfilesConfig.Producer { deriveWeightProperties(compiled); } + private void deriveFeatureDeclarations(Collection<ReferenceNode> features, + TypeContext typeContext, + Map<String, String> functionProperties) + { + for (ReferenceNode feature : features) { + var tt = feature.type(typeContext); + if (tt != null && tt.rank() > 0) { + String k = "vespa.type.feature." + feature.getName() + feature.getArguments(); + functionProperties.put(k, tt.toString()); + } + } + } + private void deriveFilterFields(RankProfile rp) { filterFields.addAll(rp.allFilterFields()); } diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg index b28843736d8..4600884b615 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -46,6 +46,8 @@ rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" rankprofile[].fef.property[].value "query(yy)" rankprofile[].fef.property[].name "rankingExpression(handicap).type" rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.feature.attribute(aa)" +rankprofile[].fef.property[].value "tensor(d1[3])" rankprofile[].fef.property[].name "rankingExpression(indirect_a).rankingScript" rankprofile[].fef.property[].value "attribute(aa)" rankprofile[].fef.property[].name "rankingExpression(indirect_a).type" diff --git a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg index 42ddad8abe7..37d84c1a2d9 100644 --- a/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_token_functions/rank-profiles.cfg @@ -13,6 +13,8 @@ rankprofile[].fef.property[].value "true" rankprofile[].fef.property[].name "vespa.type.attribute.tokens" rankprofile[].fef.property[].value "tensor(d0[128])" rankprofile[].name "using_model" +rankprofile[].fef.property[].name "vespa.type.feature.attribute(tokens)" +rankprofile[].fef.property[].value "tensor(d0[128])" rankprofile[].fef.property[].name "rankingExpression(__token_length@4d7c1b66085df918).rankingScript" rankprofile[].fef.property[].value "reduce(map(query(input), f(x)(x > 0)), sum)" rankprofile[].fef.property[].name "rankingExpression(__token_length@a16087c578950aea).rankingScript" diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index c5daf67332d..b0f7d0f2477 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -347,6 +347,8 @@ rankprofile[].fef.property[].value "3000 * rankingExpression(m2) * rankingExpres rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].name "withmf" +rankprofile[].fef.property[].name "vespa.type.feature.mymul" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "rankingExpression(myplus).rankingScript" rankprofile[].fef.property[].value "attribute(foo1) + attribute(foo2)" rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" @@ -405,6 +407,8 @@ rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" rankprofile[].fef.property[].value "attribute(t1) * query(fromq)" rankprofile[].fef.property[].name "rankingExpression(mymul).type" rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.feature.attribute(t1)" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "attribute(foo1)" rankprofile[].fef.property[].name "vespa.rank.globalphase" @@ -456,6 +460,8 @@ rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)" rankprofile[].fef.property[].name "rankingExpression(mymul).type" rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.feature.mymul" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "attribute(foo1)" rankprofile[].fef.property[].name "vespa.rank.globalphase" @@ -474,3 +480,43 @@ rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "vespa.type.query.v" rankprofile[].fef.property[].value "tensor(v[3])" +rankprofile[].name "withtfl" +rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" +rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)" +rankprofile[].fef.property[].name "rankingExpression(mymul).type" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.feature.tensorFromLabels(attribute(labels),m)" +rankprofile[].fef.property[].value "tensor(m{})" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(rankingExpression(mymul), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.query.v" +rankprofile[].fef.property[].value "tensor(v[3])" +rankprofile[].name "withtfl2" +rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" +rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m) * query(v)" +rankprofile[].fef.property[].name "rankingExpression(mymul).type" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.feature.tensorFromLabels(attribute(labels),m)" +rankprofile[].fef.property[].value "tensor(m{})" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "attribute(foo1)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(rankingExpression(mymul), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)" +rankprofile[].fef.property[].name "vespa.hidden.matchfeature" +rankprofile[].fef.property[].value "tensorFromLabels(attribute(labels),m)" +rankprofile[].fef.property[].name "vespa.type.attribute.t1" +rankprofile[].fef.property[].value "tensor(m{},v[3])" +rankprofile[].fef.property[].name "vespa.type.query.v" +rankprofile[].fef.property[].value "tensor(v[3])" diff --git a/config-model/src/test/derived/rankingexpression/rankexpression.sd b/config-model/src/test/derived/rankingexpression/rankexpression.sd index 4b3bd516619..b0de2c60299 100644 --- a/config-model/src/test/derived/rankingexpression/rankexpression.sd +++ b/config-model/src/test/derived/rankingexpression/rankexpression.sd @@ -408,4 +408,37 @@ schema rankexpression { } } + rank-profile withtfl { + inputs { + query(v) tensor(v[3]) + } + function mymul() { + expression: tensorFromLabels(attribute(labels), m) * query(v) + } + first-phase { + expression: attribute(foo1) + } + global-phase { + expression: sum(mymul) + } + match-features { + tensorFromLabels(attribute(labels), m) + } + } + + rank-profile withtfl2 { + inputs { + query(v) tensor(v[3]) + } + function mymul() { + expression: tensorFromLabels(attribute(labels), m) * query(v) + } + first-phase { + expression: attribute(foo1) + } + global-phase { + expression: sum(mymul) + } + } + } diff --git a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg index ea2d051484b..b6f02133680 100644 --- a/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg +++ b/config-model/src/test/derived/renamedfeatures/rank-profiles.cfg @@ -54,6 +54,8 @@ rankprofile[].fef.property[].value "myplus" rankprofile[].fef.property[].name "vespa.type.attribute.t1" rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].name "withmf" +rankprofile[].fef.property[].name "vespa.type.feature.mymul" +rankprofile[].fef.property[].value "tensor(m{},v[3])" rankprofile[].fef.property[].name "rankingExpression(mymul).rankingScript" rankprofile[].fef.property[].value "attribute(t1) * query(fromq)" rankprofile[].fef.property[].name "rankingExpression(mymul).type" diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java index b094130ed0f..bd649645a35 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java @@ -152,13 +152,15 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("onnx(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); assertEquals("test_summary_features", config.rankprofile(4).name()); - assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name()); - assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name()); - assertEquals("1", config.rankprofile(4).fef().property(3).value()); - assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name()); - assertEquals("onnx(another_model).out", config.rankprofile(4).fef().property(4).value()); - assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name()); - assertEquals("onnx(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(5).value()); + assertEquals("vespa.type.feature.onnx(another_model)", config.rankprofile(4).fef().property(0).name()); + assertEquals("tensor<float>(d0[2])", config.rankprofile(4).fef().property(0).value()); + assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(2).name()); + assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(5).name()); + assertEquals("1", config.rankprofile(4).fef().property(5).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(6).name()); + assertEquals("onnx(another_model).out", config.rankprofile(4).fef().property(6).value()); + assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(7).name()); + assertEquals("onnx(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(7).value()); assertEquals("test_dynamic_model", config.rankprofile(5).name()); assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name()); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 085f80c0ae6..ea8983d039c 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -333,6 +333,9 @@ public class RankProfilesConfigImporter { if ( ! expressionMatcher.matches()) return Optional.empty(); String name = expressionMatcher.group(1); String argument = expressionMatcher.group(2); + if (name.equals("feature")) { + return Optional.of(argument); + } return Optional.of(name + "(" + argument + ")"); } |