Merge pull request #31358 from vespa-engine/balder/add-opportunistic-weakand-searcher

Add an opportunistic weakand searcher that will first replace the wea…

7 files changed, 148 insertions, 9 deletions

diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java
index ea43f5c8124..b782366655f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java
@@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase {
         assertTrue(chain.phases().isEmpty());
         assertEquals(1, chain.inherits().size());
         assertEquals("native", chain.inherits(0));
-        assertEquals(10, chain.components().size());
+        assertEquals(11, chain.components().size());
         assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0));
         assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1));
         assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2));
@@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase {
         assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7));
         assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8));
         assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9));
+        assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10));
         assertTrue(chain.excludes().isEmpty());
         assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type());
     }
 
     @Test
     public void require_all_default_chains_are_correct() {
-        assertEquals(63, chainsConfig.components().size());
+        assertEquals(64, chainsConfig.components().size());
         assertEquals(10, chainsConfig.chains().size());
         validateVespaPhasesChain(findChain("vespaPhases"));
         validateNativeChain(findChain("native"));
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 1c6c773afd9..1bed32c6cdf 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -8917,6 +8917,18 @@
     ],
     "fields" : [ ]
   },
+  "com.yahoo.search.searchers.OpportunisticWeakAndSearcher" : {
+    "superClass" : "com.yahoo.search.Searcher",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public"
+    ],
+    "methods" : [
+      "public void <init>()",
+      "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)"
+    ],
+    "fields" : [ ]
+  },
   "com.yahoo.search.searchers.QueryValidator" : {
     "superClass" : "com.yahoo.search.Searcher",
     "interfaces" : [ ],
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
index 7536e74042c..72a1a7d3430 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java
@@ -12,6 +12,7 @@ import com.yahoo.search.Searcher;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.search.yql.MinimalQueryInserter;
 import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Provides;
 
 /**
  * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true.
@@ -19,10 +20,12 @@ import com.yahoo.yolean.chain.After;
  *
  * @author karowan
  */
+@Provides(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND)
 @After(MinimalQueryInserter.EXTERNAL_YQL)
 public class WeakAndReplacementSearcher extends Searcher {
+    public static final String REPLACE_OR_WITH_WEAKAND = "replace-or-with-weakand";
     static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace");
-    static final CompoundName WAND_HITS = CompoundName.from("wand.hits");
+    public static final CompoundName WAND_HITS = CompoundName.from("wand.hits");
 
     @Override public Result search(Query query, Execution execution) {
         if (!query.properties().getBoolean(WEAKAND_REPLACE)) {
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java
index 69a1f8ec6cb..c03a74ea2c5 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java
@@ -34,7 +34,8 @@ public class VespaSearchers {
                     com.yahoo.prelude.searcher.PosSearcher.class,
                     com.yahoo.prelude.semantics.SemanticSearcher.class,
                     com.yahoo.search.grouping.GroupingQueryParser.class,
-                    com.yahoo.search.querytransform.WeakAndReplacementSearcher.class);
+                    com.yahoo.search.querytransform.WeakAndReplacementSearcher.class,
+                    com.yahoo.search.searchers.OpportunisticWeakAndSearcher.class);
 
     public static final Collection<ChainedComponentModel> nativeSearcherModels;
 
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java
new file mode 100644
index 00000000000..d871dda2aa2
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java
@@ -0,0 +1,83 @@
+package com.yahoo.search.searchers;
+
+import com.yahoo.api.annotations.Beta;
+import com.yahoo.component.chain.dependencies.After;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.Searcher;
+import com.yahoo.search.querytransform.WeakAndReplacementSearcher;
+import com.yahoo.search.searchchain.Execution;
+
+import static com.yahoo.search.querytransform.WeakAndReplacementSearcher.WAND_HITS;
+
+/**
+ * Will opportunistically replace the WeakAND with an AND as it is faster.
+ * If enough hits are returned all is good and we return. If not we fall back to the original query.
+ *
+ * @author baldersheim
+ */
+@Beta
+@After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND)
+public class OpportunisticWeakAndSearcher extends Searcher {
+    static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and");
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        if (!query.properties().getBoolean(OPPORTUNISTIC_AND)) {
+            return execution.search(query);
+        }
+
+        Item originalRoot = query.getModel().getQueryTree().getRoot();
+        int targetHits = targetHits(originalRoot);
+        if (targetHits >= 0) {
+            query.getModel().getQueryTree().setRoot(weakAnd2AndRecurse(originalRoot.clone()));
+            query.trace("WeakAND => AND", true, 2);
+            Result result = execution.search(query);
+            if (result.getHitCount() >= query.properties().getInteger(WAND_HITS)) {
+                return result;
+            }
+            query.getModel().getQueryTree().setRoot(originalRoot);
+            return execution.search(query);
+        }
+        return execution.search(query);
+    }
+
+    // returns targetHits for the first WeakAndItem found, -1 if none found.
+    static int targetHits(Item item) {
+        if (!(item instanceof CompositeItem compositeItem)) return -1;
+        if (item instanceof WeakAndItem weakAndItem) return weakAndItem.getN();
+        for (int i = 0; i < compositeItem.getItemCount(); i++) {
+            int targetHits = targetHits(compositeItem.getItem(i));
+            if (targetHits >= 0) return targetHits;
+        }
+        return -1;
+    }
+
+    static Item weakAnd2AndRecurse(Item item) {
+        if (!(item instanceof CompositeItem compositeItem)) return item;
+        compositeItem = weakAnd2And(compositeItem);
+        for (int i = 0; i < compositeItem.getItemCount(); i++) {
+            Item subItem = compositeItem.getItem(i);
+            Item replacedItem = weakAnd2AndRecurse(subItem);
+            if (replacedItem != subItem) {
+                compositeItem.setItem(i, replacedItem);
+            }
+        }
+        return compositeItem;
+    }
+
+    private static CompositeItem weakAnd2And(CompositeItem item) {
+        if (item instanceof WeakAndItem weakAndItem) {
+            AndItem andItem = new AndItem();
+            andItem.setWeight(weakAndItem.getWeight());
+            item.items().forEach(andItem::addItem);
+            return andItem;
+        }
+        return item;
+    }
+}
diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java
index 52f5fd0cafb..7b91a5d3c25 100644
--- a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java
@@ -23,6 +23,7 @@ import static com.yahoo.search.querytransform.WeakAndReplacementSearcher.WAND_HI
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class WeakAndReplacementSearcherTestCase {
@@ -57,7 +58,7 @@ public class WeakAndReplacementSearcherTestCase {
         Result result = buildExec().search(query);
         Item root = TestUtils.getQueryTreeRoot(result);
         assertFalse(orItemsExist(root));
-        assertTrue(root instanceof WeakAndItem);
+        assertInstanceOf(WeakAndItem.class, root);
         assertEquals(N, ((WeakAndItem) root).getN());
     }
 
@@ -103,24 +104,22 @@ public class WeakAndReplacementSearcherTestCase {
         if (item1 != item2) {
             return false;
         }
-        if (!(item1 instanceof CompositeItem)) {
+        if (!(item1 instanceof CompositeItem compositeItem1)) {
             return true;
         }
 
-        CompositeItem compositeItem1 = (CompositeItem) item1;
         CompositeItem compositeItem2 = (CompositeItem) item2;
         return IntStream.range(0, compositeItem1.getItemCount())
                 .allMatch(i -> deepEquals(compositeItem1.getItem(i), compositeItem2.getItem(i)));
     }
 
     private boolean orItemsExist(Item item) {
-        if (!(item instanceof CompositeItem)) {
+        if (!(item instanceof CompositeItem compositeItem)) {
             return false;
         }
         if (item instanceof OrItem) {
             return true;
         }
-        CompositeItem compositeItem = (CompositeItem) item;
         return compositeItem.items().stream().anyMatch(this::orItemsExist);
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java
new file mode 100644
index 00000000000..642b9bc7bc4
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java
@@ -0,0 +1,40 @@
+package com.yahoo.search.searchers;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordItem;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+public class OpportunisticWeakAndSearcherTestCase {
+    private static Item buildQueryItem(CompositeItem root, CompositeItem injectAtLevel2) {
+        root.addItem(new WordItem("text"));
+        injectAtLevel2.addItem(new WordItem("a"));
+        injectAtLevel2.addItem(new WordItem("b"));
+        root.addItem(injectAtLevel2);
+        return root;
+    }
+
+    @Test
+    public void requireThatWeakAndIsDetected() {
+        assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(new OrItem()));
+        assertEquals(33, OpportunisticWeakAndSearcher.targetHits(new WeakAndItem(33)));
+        assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new WeakAndItem(77))));
+        assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new AndItem(), new WeakAndItem(77))));
+        assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new AndItem())));
+    }
+
+    @Test
+    public void requireThatWeakAndIsReplacedWithAnd() {
+        assertEquals(buildQueryItem(new OrItem(), new AndItem()),
+                OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem())));
+        assertEquals(buildQueryItem(new AndItem(), new AndItem()),
+                OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem())));
+    }
+
+}