Merge pull request #28054 from vespa-engine/geirst/target-hits-max-adjustment-factor

Control the auto-adjustment of targetHits in ANN using post-filtering.

4 files changed, 23 insertions, 3 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 0f440957dfd..cdb660f294a 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -6981,12 +6981,14 @@
       "public java.lang.Integer getMinHitsPerThread()",
       "public java.lang.Double getPostFilterThreshold()",
       "public java.lang.Double getApproximateThreshold()",
+      "public java.lang.Double getTargetHitsMaxAdjustmentFactor()",
       "public void setTermwiselimit(double)",
       "public void setNumThreadsPerSearch(int)",
       "public void setNumSearchPartitions(int)",
       "public void setMinHitsPerThread(int)",
       "public void setPostFilterThreshold(double)",
       "public void setApproximateThreshold(double)",
+      "public void setTargetHitsMaxAdjustmentFactor(double)",
       "public void prepare(com.yahoo.search.query.ranking.RankProperties)",
       "public com.yahoo.search.query.ranking.Matching clone()",
       "public boolean equals(java.lang.Object)",
@@ -7000,6 +7002,7 @@
       "public static final java.lang.String MINHITSPERTHREAD",
       "public static final java.lang.String POST_FILTER_THRESHOLD",
       "public static final java.lang.String APPROXIMATE_THRESHOLD",
+      "public static final java.lang.String TARGET_HITS_MAX_ADJUSTMENT_FACTOR",
       "public java.lang.Double termwiseLimit"
     ]
   },
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 800b3a1ba89..99d6959441a 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -91,6 +91,7 @@ public class QueryProperties extends Properties {
         addDualCasedRM(map, Matching.MINHITSPERTHREAD, GetterSetter.of(query -> query.getRanking().getMatching().getMinHitsPerThread(), (query, value) -> query.getRanking().getMatching().setMinHitsPerThread(asInteger(value, 0))));
         addDualCasedRM(map, Matching.POST_FILTER_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getPostFilterThreshold(), (query, value) -> query.getRanking().getMatching().setPostFilterThreshold(asDouble(value, 1.0))));
         addDualCasedRM(map, Matching.APPROXIMATE_THRESHOLD, GetterSetter.of(query -> query.getRanking().getMatching().getApproximateThreshold(), (query, value) -> query.getRanking().getMatching().setApproximateThreshold(asDouble(value, 0.05))));
+        addDualCasedRM(map, Matching.TARGET_HITS_MAX_ADJUSTMENT_FACTOR, GetterSetter.of(query -> query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor(), (query, value) -> query.getRanking().getMatching().setTargetHitsMaxAdjustmentFactor(asDouble(value, 20.0))));
 
         map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ATTRIBUTE), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAttribute(), (query, value) -> query.getRanking().getMatchPhase().setAttribute(asString(value, null))));
         map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, MatchPhase.ASCENDING), GetterSetter.of(query -> query.getRanking().getMatchPhase().getAscending(), (query, value) -> query.getRanking().getMatchPhase().setAscending(asBoolean(value, false))));
diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java
index 35fbd52f967..4d21f32d16d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java
+++ b/container-search/src/main/java/com/yahoo/search/query/ranking/Matching.java
@@ -24,6 +24,7 @@ public class Matching implements Cloneable {
     public static final String MINHITSPERTHREAD = "minHitsPerThread";
     public static final String POST_FILTER_THRESHOLD = "postFilterThreshold";
     public static final String APPROXIMATE_THRESHOLD = "approximateThreshold";
+    public static final String TARGET_HITS_MAX_ADJUSTMENT_FACTOR = "targetHitsMaxAdjustmentFactor";
 
     static {
         argumentType =new QueryProfileType(Ranking.MATCHING);
@@ -35,6 +36,7 @@ public class Matching implements Cloneable {
         argumentType.addField(new FieldDescription(MINHITSPERTHREAD, "integer"));
         argumentType.addField(new FieldDescription(POST_FILTER_THRESHOLD, "double"));
         argumentType.addField(new FieldDescription(APPROXIMATE_THRESHOLD, "double"));
+        argumentType.addField(new FieldDescription(TARGET_HITS_MAX_ADJUSTMENT_FACTOR, "double"));
         argumentType.freeze();
     }
 
@@ -46,6 +48,7 @@ public class Matching implements Cloneable {
     private Integer minHitsPerThread = null;
     private Double postFilterThreshold = null;
     private Double approximateThreshold = null;
+    private Double targetHitsMaxAdjustmentFactor = null;
 
     public Double getTermwiseLimit() { return termwiseLimit; }
     public Integer getNumThreadsPerSearch() { return numThreadsPerSearch; }
@@ -53,6 +56,7 @@ public class Matching implements Cloneable {
     public Integer getMinHitsPerThread() { return minHitsPerThread; }
     public Double getPostFilterThreshold() { return postFilterThreshold; }
     public Double getApproximateThreshold() { return approximateThreshold; }
+    public Double getTargetHitsMaxAdjustmentFactor() { return targetHitsMaxAdjustmentFactor; }
 
     public void setTermwiselimit(double value) {
         if ((value < 0.0) || (value > 1.0)) {
@@ -75,6 +79,9 @@ public class Matching implements Cloneable {
     public void setApproximateThreshold(double threshold) {
         approximateThreshold = threshold;
     }
+    public void setTargetHitsMaxAdjustmentFactor(double factor) {
+        targetHitsMaxAdjustmentFactor = factor;
+    }
 
     /** Internal operation - DO NOT USE */
     public void prepare(RankProperties rankProperties) {
@@ -97,6 +104,9 @@ public class Matching implements Cloneable {
         if (approximateThreshold != null) {
             rankProperties.put("vespa.matching.global_filter.lower_limit", String.valueOf(approximateThreshold));
         }
+        if (targetHitsMaxAdjustmentFactor != null) {
+            rankProperties.put("vespa.matching.nns.target_hits_max_adjustment_factor", String.valueOf(targetHitsMaxAdjustmentFactor));
+        }
     }
 
     @Override
@@ -119,12 +129,14 @@ public class Matching implements Cloneable {
                 Objects.equals(numSearchPartitions, matching.numSearchPartitions) &&
                 Objects.equals(minHitsPerThread, matching.minHitsPerThread) &&
                 Objects.equals(postFilterThreshold, matching.postFilterThreshold) &&
-                Objects.equals(approximateThreshold, matching.approximateThreshold);
+                Objects.equals(approximateThreshold, matching.approximateThreshold) &&
+                Objects.equals(targetHitsMaxAdjustmentFactor, matching.targetHitsMaxAdjustmentFactor);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(termwiseLimit, numThreadsPerSearch, numSearchPartitions, minHitsPerThread, postFilterThreshold, approximateThreshold);
+        return Objects.hash(termwiseLimit, numThreadsPerSearch, numSearchPartitions, minHitsPerThread,
+                postFilterThreshold, approximateThreshold, targetHitsMaxAdjustmentFactor);
     }
 }
 
diff --git a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java
index e3a1eb18a33..37d0e9e1072 100644
--- a/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/MatchingTestCase.java
@@ -20,6 +20,7 @@ public class MatchingTestCase {
         assertNull(query.getRanking().getMatching().getMinHitsPerThread());
         assertNull(query.getRanking().getMatching().getPostFilterThreshold());
         assertNull(query.getRanking().getMatching().getApproximateThreshold());
+        assertNull(query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor());
     }
 
     @Test
@@ -30,13 +31,15 @@ public class MatchingTestCase {
                 "&ranking.matching.numSearchPartitions=13" +
                 "&ranking.matching.minHitsPerThread=3" +
                 "&ranking.matching.postFilterThreshold=0.8" +
-                "&ranking.matching.approximateThreshold=0.3");
+                "&ranking.matching.approximateThreshold=0.3" +
+                "&ranking.matching.targetHitsMaxAdjustmentFactor=2.5");
         assertEquals(Double.valueOf(0.7), query.getRanking().getMatching().getTermwiseLimit());
         assertEquals(Integer.valueOf(17), query.getRanking().getMatching().getNumThreadsPerSearch());
         assertEquals(Integer.valueOf(13), query.getRanking().getMatching().getNumSearchPartitions());
         assertEquals(Integer.valueOf(3), query.getRanking().getMatching().getMinHitsPerThread());
         assertEquals(Double.valueOf(0.8), query.getRanking().getMatching().getPostFilterThreshold());
         assertEquals(Double.valueOf(0.3), query.getRanking().getMatching().getApproximateThreshold());
+        assertEquals(Double.valueOf(2.5), query.getRanking().getMatching().getTargetHitsMaxAdjustmentFactor());
 
         query.prepare();
         assertEquals("0.7", query.getRanking().getProperties().get("vespa.matching.termwise_limit").get(0));
@@ -45,6 +48,7 @@ public class MatchingTestCase {
         assertEquals("3", query.getRanking().getProperties().get("vespa.matching.minhitsperthread").get(0));
         assertEquals("0.8", query.getRanking().getProperties().get("vespa.matching.global_filter.upper_limit").get(0));
         assertEquals("0.3", query.getRanking().getProperties().get("vespa.matching.global_filter.lower_limit").get(0));
+        assertEquals("2.5", query.getRanking().getProperties().get("vespa.matching.nns.target_hits_max_adjustment_factor").get(0));
     }
 
     @Test