add match-features in config-model

* extend parser with single-line and multi-line "match-features"
* add match features in RankProfile
* rewrite match features if necessary
* generate config for the backend

New code is similar to the way "summary-features" and "rank-features" are handled.

3 files changed, 73 insertions, 20 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index ad3369b6d2d..b5d86f5bb38 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -97,6 +97,7 @@ public class RankProfile implements Cloneable {
     private Set<ReferenceNode> summaryFeatures;
     private String inheritedSummaryFeatures;
 
+    private Set<ReferenceNode> matchFeatures;
     private Set<ReferenceNode> rankFeatures;
 
     /** The properties of this - a multimap */
@@ -518,6 +519,26 @@ public class RankProfile implements Cloneable {
         this.inheritedSummaryFeatures = parentProfile;
     }
 
+    /** Returns a read-only view of the match features to use in this profile. This is never null */
+    public Set<ReferenceNode> getMatchFeatures() {
+        if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures);
+        if (getInherited() != null) return getInherited().getMatchFeatures();
+        return Set.of();
+    }
+
+    private void addMatchFeature(ReferenceNode feature) {
+        if (matchFeatures == null)
+            matchFeatures = new LinkedHashSet<>();
+        matchFeatures.add(feature);
+    }
+
+    /** Adds the content of the given feature list to the internal list of summary features. */
+    public void addMatchFeatures(FeatureList features) {
+        for (ReferenceNode feature : features) {
+            addMatchFeature(feature);
+        }
+    }
+
     /** Returns a read-only view of the rank features to use in this profile. This is never null */
     public Set<ReferenceNode> getRankFeatures() {
         if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures);
@@ -817,6 +838,7 @@ public class RankProfile implements Cloneable {
             clone.rankSettings = new LinkedHashSet<>(this.rankSettings);
             clone.matchPhaseSettings = this.matchPhaseSettings; // hmm?
             clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null;
+            clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null;
             clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null;
             clone.rankProperties = new LinkedHashMap<>(this.rankProperties);
             clone.inputFeatures = new LinkedHashMap<>(this.inputFeatures);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 9d8ca06f77a..a33498a37ec 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -49,6 +49,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
 
     // TODO: These are to expose coupling between the strings used here and elsewhere
     public static final String summaryFeatureFefPropertyPrefix = "vespa.summary.feature";
+    public static final String matchFeatureFefPropertyPrefix = "vespa.match.feature";
     public static final String rankFeatureFefPropertyPrefix = "vespa.dump.feature";
 
     private final String name;
@@ -121,6 +122,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
 
         private final Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>();
         private final Set<ReferenceNode> summaryFeatures;
+        private final Set<ReferenceNode> matchFeatures;
         private final Set<ReferenceNode> rankFeatures;
         private final List<RankProfile.RankProperty> rankProperties;
 
@@ -163,6 +165,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             firstPhaseRanking = compiled.getFirstPhaseRanking();
             secondPhaseRanking = compiled.getSecondPhaseRanking();
             summaryFeatures = new LinkedHashSet<>(compiled.getSummaryFeatures());
+            matchFeatures = new LinkedHashSet<>(compiled.getMatchFeatures());
             rankFeatures = compiled.getRankFeatures();
             rerankCount = compiled.getRerankCount();
             matchPhaseSettings = compiled.getMatchPhaseSettings();
@@ -188,8 +191,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                 functionProperties.putAll(secondPhaseRanking.getRankProperties(functionSerializationContext));
             }
 
-            derivePropertiesAndSummaryFeaturesFromFunctions(functions, functionProperties, functionSerializationContext);
-            deriveOnnxModelFunctionsAndSummaryFeatures(compiled);
+            derivePropertiesAndFeaturesFromFunctions(functions, functionProperties, functionSerializationContext);
+            deriveOnnxModelFunctionsAndFeatures(compiled);
 
             deriveRankTypeSetting(compiled, attributeFields);
             deriveFilterFields(compiled);
@@ -200,12 +203,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             filterFields.addAll(rp.allFilterFields());
         }
 
-        private void derivePropertiesAndSummaryFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions,
+        private void derivePropertiesAndFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions,
                                                                      Map<String, String> functionProperties,
                                                                      SerializationContext functionContext) {
             if (functions.isEmpty()) return;
 
-            replaceFunctionSummaryFeatures(functionContext);
+            replaceFunctionFeatures(summaryFeatures, functionContext);
+            replaceFunctionFeatures(matchFeatures, functionContext);
 
             // First phase, second phase and summary features should add all required functions to the context.
             // However, we need to add any functions not referenced in those anyway for model-evaluation.
@@ -236,10 +240,10 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             functionProperties.putAll(context.serializedFunctions());
         }
 
-        private void replaceFunctionSummaryFeatures(SerializationContext context) {
-            if (summaryFeatures == null) return;
-            Map<String, ReferenceNode> functionSummaryFeatures = new LinkedHashMap<>();
-            for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) {
+        private void replaceFunctionFeatures(Set<ReferenceNode> features, SerializationContext context) {
+            if (features == null) return;
+            Map<String, ReferenceNode> functionFeatures = new LinkedHashMap<>();
+            for (Iterator<ReferenceNode> i = features.iterator(); i.hasNext(); ) {
                 ReferenceNode referenceNode = i.next();
                 // Is the feature a function?
                 ExpressionFunction function = context.getFunction(referenceNode.getName());
@@ -248,13 +252,13 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
                     String expressionString = function.getBody().getRoot().toString(context).toString();
                     context.addFunctionSerialization(propertyName, expressionString);
                     ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput());
-                    functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode);
+                    functionFeatures.put(referenceNode.getName(), newReferenceNode);
                     i.remove(); // Will add the expanded one in next block
                 }
             }
-            // Then, replace the summary features that were functions
-            for (Map.Entry<String, ReferenceNode> e : functionSummaryFeatures.entrySet()) {
-                summaryFeatures.add(e.getValue());
+            // Then, replace the features that were functions
+            for (Map.Entry<String, ReferenceNode> e : functionFeatures.entrySet()) {
+                features.add(e.getValue());
             }
         }
 
@@ -352,6 +356,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             for (ReferenceNode feature : summaryFeatures) {
                 properties.add(new Pair<>(summaryFeatureFefPropertyPrefix, feature.toString()));
             }
+            for (ReferenceNode feature : matchFeatures) {
+                properties.add(new Pair<>(matchFeatureFefPropertyPrefix, feature.toString()));
+            }
             for (ReferenceNode feature : rankFeatures) {
                 properties.add(new Pair<>(rankFeatureFefPropertyPrefix, feature.toString()));
             }
@@ -441,11 +448,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             return properties;
         }
 
-        private void deriveOnnxModelFunctionsAndSummaryFeatures(RankProfile rankProfile) {
+        private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) {
             if (rankProfile.getSearch() == null) return;
             if (rankProfile.getSearch().onnxModels().asMap().isEmpty()) return;
             replaceOnnxFunctionInputs(rankProfile);
-            replaceImplicitOnnxConfigSummaryFeatures(rankProfile);
+            replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile);
+            replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile);
         }
 
         private void replaceOnnxFunctionInputs(RankProfile rankProfile) {
@@ -461,18 +469,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
             }
         }
 
-        private void replaceImplicitOnnxConfigSummaryFeatures(RankProfile rankProfile) {
-            if (summaryFeatures == null || summaryFeatures.isEmpty()) return;
-            Set<ReferenceNode> replacedSummaryFeatures = new HashSet<>();
-            for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) {
+        private void replaceImplicitOnnxConfigFeatures(Set<ReferenceNode> features, RankProfile rankProfile) {
+            if (features == null || features.isEmpty()) return;
+            Set<ReferenceNode> replacedFeatures = new HashSet<>();
+            for (Iterator<ReferenceNode> i = features.iterator(); i.hasNext(); ) {
                 ReferenceNode referenceNode = i.next();
                 ReferenceNode replacedNode = (ReferenceNode) OnnxModelTransformer.transformFeature(referenceNode, rankProfile);
                 if (referenceNode != replacedNode) {
-                    replacedSummaryFeatures.add(replacedNode);
+                    replacedFeatures.add(replacedNode);
                     i.remove();
                 }
             }
-            summaryFeatures.addAll(replacedSummaryFeatures);
+            features.addAll(replacedFeatures);
         }
 
     }
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 393a63c122a..fe522494baf 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -355,6 +355,8 @@ TOKEN :
 | < DISTANCEMETRIC: "distance-metric" >
 | < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
 | < MULTITHREADEDINDEXING: "multi-threaded-indexing" >
+| < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? >
+| < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
 | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
 | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
 | < SUMMARYFEATURES_ML_INHERITS: "summary-features inherits " (<IDENTIFIER>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
@@ -2078,6 +2080,7 @@ Object rankProfileItem(RankProfile profile) : { }
       | secondPhase(profile)
       | rankDegradation(profile)
       | constants(profile)
+      | matchFeatures(profile)
       | summaryFeatures(profile) )
     { return null; }
 }
@@ -2319,6 +2322,26 @@ Object summaryFeatures(RankProfile profile) :
     }
 }
 
+/**
+ * This rule consumes a match-features block of a rank profile.
+ *
+ * @param profile The rank profile to modify.
+ * @return Null.
+ */
+Object matchFeatures(RankProfile profile) :
+{
+    String features;
+}
+{
+    ( <MATCHFEATURES_SL> { features = token.image.substring(token.image.indexOf(":") + 1).trim(); } |
+      <MATCHFEATURES_ML> { features = token.image.substring(token.image.indexOf("{") + 1,
+                                                              token.image.lastIndexOf("}")).trim(); } ) 
+    {
+        profile.addMatchFeatures(getFeatureList(features));
+        return null;
+    }
+}
+
 /** Consumes a rank-features block of a rank profile */
 Object rankFeatures(RankProfile profile) :
 {