diff options
Diffstat (limited to 'container-search')
11 files changed, 222 insertions, 32 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java index a70d653b90a..89e85fe8f12 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WandItem.java @@ -17,7 +17,7 @@ import java.nio.ByteBuffer; */ public class WandItem extends WeightedSetItem { - private int targetNumHits; + private final int targetNumHits; private double scoreThreshold = 0; private double thresholdBoostFactor = 1; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 033986dc90f..4fa2ed8b214 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -18,7 +18,7 @@ import java.nio.ByteBuffer; */ public final class WeakAndItem extends NonReducibleCompositeItem { - private int N; + private int n; private String index; private int scoreThreshold = 0; @@ -31,15 +31,15 @@ public final class WeakAndItem extends NonReducibleCompositeItem { } /** - * Make a WAND item with no children. You can mention a common index or you can mention it on each child. + * Make a WeakAnd item with no children. You can mention a common index or you can mention it on each child. * - * @param index The index it shall search. - * @param N the target for minimum number of hits to produce; + * @param index the index to search + * @param n the target for minimum number of hits to produce; * a backend will not suppress any hits in the operator * until N hits have been produced. - **/ - public WeakAndItem(String index, int N) { - this.N = N; + */ + public WeakAndItem(String index, int n) { + this.n = n; this.index = (index == null) ? "" : index; } public WeakAndItem(int N) { @@ -61,7 +61,7 @@ public final class WeakAndItem extends NonReducibleCompositeItem { protected void appendHeadingString(StringBuilder buffer) { buffer.append(getName()); buffer.append("("); - buffer.append(N); + buffer.append(n); buffer.append(")"); buffer.append(" "); } @@ -75,53 +75,52 @@ public final class WeakAndItem extends NonReducibleCompositeItem { } public int getN() { - return N; + return n; } public void setN(int N) { - this.N = N; + this.n = N; } + @Deprecated // TODO: Remove on Vespa 8 public int getScoreThreshold() { return scoreThreshold; } /** - * Sets the score threshold used by the backend search operator handling this WeakAndItem. - * This threshold is currently only used if the WeakAndItem is searching a RISE index field. - * The score threshold then specifies the minimum dot product score a match needs to be part of the result set. - * Default value is 0. + * Noop. * - * @param scoreThreshold the score threshold. + * @deprecated has no effect */ + @Deprecated // TODO: Remove on Vespa 8 public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); - IntegerCompressor.putCompressedPositiveNumber(N, buffer); + IntegerCompressor.putCompressedPositiveNumber(n, buffer); putString(index, buffer); } @Override public void disclose(Discloser discloser) { super.disclose(discloser); - discloser.addProperty("N", N); + discloser.addProperty("N", n); } + @Override public int hashCode() { - return super.hashCode() + 31 * N; + return super.hashCode() + 31 * n; } - /** - * Returns whether this item is of the same class and - * contains the same state as the given item - */ + /** Returns whether this item is of the same class and contains the same state as the given item. */ + @Override public boolean equals(Object object) { if (!super.equals(object)) return false; WeakAndItem other = (WeakAndItem) object; // Ensured by superclass - if (this.N != other.N) return false; + if (this.n != other.n) return false; return true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index e2dc5df5242..74a993b0413 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -148,10 +148,10 @@ public class AdvancedParser extends StructuredParser { } return topLevelItem; } else if (isTheWord("wand", item)) { - int n=consumeNumericArgument(); - if (n==0) + int n = consumeNumericArgument(); + if (n == 0) n=WeakAndItem.defaultN; - if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n!=((WeakAndItem)topLevelItem).getN()) { + if (topLevelIsClosed || !(topLevelItem instanceof WeakAndItem) || n != ((WeakAndItem)topLevelItem).getN()) { WeakAndItem wand = new WeakAndItem(); wand.setN(n); wand.addItem(topLevelItem); diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 5f1f26b77e9..d212c2d6ec4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -502,20 +502,21 @@ public class SelectParser implements Parser { return item; } + @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); Inspector annotations = getAnnotations(value); - if (annotations != null){ + if (annotations != null) { annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { if (TARGET_HITS.equals(annotation_name)){ weakAnd.setN((int)(annotation_value.asDouble())); } - if (TARGET_NUM_HITS.equals(annotation_name)){ + if (TARGET_NUM_HITS.equals(annotation_name)) { weakAnd.setN((int)(annotation_value.asDouble())); } - if (SCORE_THRESHOLD.equals(annotation_name)){ + if (SCORE_THRESHOLD.equals(annotation_name)) { weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); } }); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java new file mode 100644 index 00000000000..058f6b93ae3 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -0,0 +1,66 @@ +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.*; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. + * Otherwise a noop searcher. + * + * @author karowan + */ +public class WeakAndReplacementSearcher extends Searcher { + private static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); + + @Override public Result search(Query query, Execution execution) { + if (!query.properties().getBoolean(WEAKAND_REPLACE)) { + return execution.search(query); + } + replaceOrItems(query); + return execution.search(query); + } + + /** + * Extracts the queryTree root and the wand.hits property to send to the recursive replacement function + * @param query the search query + */ + private void replaceOrItems(Query query) { + Item root = query.getModel().getQueryTree().getRoot(); + int hits = query.properties().getInteger("wand.hits", WeakAndItem.defaultN); + query.getModel().getQueryTree().setRoot(replaceOrItems(root, hits)); + if (root != query.getModel().getQueryTree().getRoot()) + query.trace("Replaced OR by WeakAnd", true, 2); + } + + + /** + * Recursively iterates over an Item to replace all instances of OrItems with WeakAndItems + * @param item the current item in the replacement iteration + * @param hits the wand.hits property from the request which is assigned to the N value of the new WeakAndItem + * @return The original item or a WeakAndItem replacement of an OrItem + */ + private Item replaceOrItems(Item item, int hits) { + if (!(item instanceof CompositeItem)) { + return item; + } + CompositeItem compositeItem = (CompositeItem) item; + if (compositeItem instanceof OrItem) { + WeakAndItem newItem = new WeakAndItem(hits); + newItem.setWeight(compositeItem.getWeight()); + compositeItem.items().forEach(newItem::addItem); + compositeItem = newItem; + } + for (int i = 0; i < compositeItem.getItemCount(); i++) { + Item subItem = compositeItem.getItem(i); + Item replacedItem = replaceOrItems(subItem, hits); + if (replacedItem != subItem) { + compositeItem.setItem(i, replacedItem); + } + } + return compositeItem; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index d24134e8d88..c264425cb9c 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -34,7 +34,8 @@ public class VespaSearchers { com.yahoo.prelude.searcher.BlendingSearcher.class, com.yahoo.prelude.searcher.PosSearcher.class, com.yahoo.prelude.semantics.SemanticSearcher.class, - com.yahoo.search.grouping.GroupingQueryParser.class); + com.yahoo.search.grouping.GroupingQueryParser.class, + com.yahoo.search.querytransform.WeakAndReplacementSearcher.class); public static final Collection<ChainedComponentModel> nativeSearcherModels; diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index f4a36ea51ab..9e6701572dc 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -950,12 +950,13 @@ public class VespaSerializer { } + @SuppressWarnings("deprecation") private static class WeakAndSerializer extends Serializer<WeakAndItem> { @Override void onExit(StringBuilder destination, WeakAndItem item) { destination.append(')'); - if (needsAnnotationBlock((WeakAndItem) item)) { + if (needsAnnotationBlock(item)) { destination.append(')'); } } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index f37aeb4c1e0..9eaea47ea1b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -1099,6 +1099,7 @@ public class YqlParser implements Parser { return convertVarArgs(spec, 0, new OrItem()); } + @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(OperatorNode<ExpressionOperator> spec) { WeakAndItem weakAnd = new WeakAndItem(); Integer targetNumHits = getAnnotation(spec, TARGET_HITS, diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java new file mode 100644 index 00000000000..1aa61d7535b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java @@ -0,0 +1,119 @@ +package com.yahoo.search.querytransform; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.*; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.stream.IntStream; + +import static org.junit.Assert.*; + +public class WeakAndReplacementSearcherTestCase { + + private static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); + private static final int N = 99; + + + private Execution buildExec() { + return new Execution(new Chain<Searcher>(new WeakAndReplacementSearcher()), + Execution.Context.createContextStub()); + } + + private Query buildDefaultQuery(boolean searcherEnabled) { + Query query = new Query(); + query.properties().set("wand.hits", N); + query.properties().set(WEAKAND_REPLACE, searcherEnabled); + OrItem root = new OrItem(); + root.addItem(new WordItem("text")); + NotItem notItem = new NotItem(); + OrItem notItemOr = new OrItem(); + notItemOr.addItem(new IntItem(1, "index")); + notItemOr.addItem(new WordItem("positive")); + notItem.addPositiveItem(notItemOr); + notItem.addNegativeItem(new WordItem("negative")); + query.getModel().getQueryTree().setRoot(root); + return query; + } + + + + + @Test + public void requireOrItemsToBeReplaced() { + Query query = buildDefaultQuery(true); + Result result = buildExec().search(query); + Item root = TestUtils.getQueryTreeRoot(result); + assertFalse(orItemsExist(root)); + assertTrue(root instanceof WeakAndItem); + assertEquals(N, ((WeakAndItem)root).getN()); + } + + @Test + public void requireQueryPropertyToWork() { + Query query = buildDefaultQuery(false); + Item preRoot = query.getModel().getQueryTree().getRoot(); + Result result = buildExec().search(query); + Item root = TestUtils.getQueryTreeRoot(result); + assertTrue(orItemsExist(root)); + assertTrue(deepEquals(root, preRoot)); + } + + @Test + public void requireDoNothingOnNoOrItems() { + Query query = new Query(); + query.properties().set(WEAKAND_REPLACE, true); + AndItem andItem = new AndItem(); + andItem.addItem(new WordItem("1")); + andItem.addItem(new WordItem("2")); + query.getModel().getQueryTree().setRoot(andItem); + Result result = buildExec().search(query); + Item root = TestUtils.getQueryTreeRoot(result); + assertTrue(deepEquals(root, andItem)); + } + + @Test + public void requireChildrenAreTheSame() { + Query query = new Query(); + query.properties().set(WEAKAND_REPLACE, true); + OrItem preRoot = new OrItem(); + preRoot.addItem(new WordItem("val1")); + preRoot.addItem(new WordItem("val2")); + + query.getModel().getQueryTree().setRoot(preRoot); + Result result = buildExec().search(query); + WeakAndItem root = (WeakAndItem)TestUtils.getQueryTreeRoot(result); + assertEquals(preRoot.getItem(0), root.getItem(0)); + assertEquals(preRoot.getItem(1), root.getItem(1)); + } + + private boolean deepEquals(Item item1, Item item2) { + if (item1 != item2) { + return false; + } + if (!(item1 instanceof CompositeItem)) { + return true; + } + + CompositeItem compositeItem1 = (CompositeItem) item1; + CompositeItem compositeItem2 = (CompositeItem) item2; + return IntStream.range(0, compositeItem1.getItemCount()) + .allMatch(i -> deepEquals(compositeItem1.getItem(i), compositeItem2.getItem(i))); + } + + private boolean orItemsExist(Item item) { + if (!(item instanceof CompositeItem)) { + return false; + } + if (item instanceof OrItem) { + return true; + } + CompositeItem compositeItem = (CompositeItem) item; + return compositeItem.items().stream().anyMatch(this::orItemsExist); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 2d88351f9ea..3f0000367e7 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -616,6 +616,7 @@ public class YqlParserTestCase { } @Test + @SuppressWarnings("deprecation") public void testWeakAnd() { assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", "WAND(100) a:A b:B"); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 1ce70feebb4..7549c67d0ae 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -552,6 +552,7 @@ public class SelectTestCase { } @Test + @SuppressWarnings("deprecation") public void testWeakAnd() { assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }", "WAND(100) a:A b:B"); |