diff options
Diffstat (limited to 'container-search/src')
14 files changed, 124 insertions, 63 deletions
diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index 9af8d83c6d7..422c3c8681e 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -127,7 +127,7 @@ HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ; fragment ESC_SEQ - : '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\'|'/') + : '\\' ('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\'|'/') | UNICODE_ESC ; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index d3fbeb020f8..f48dc9a8630 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -42,13 +42,9 @@ public abstract class CompositeItem extends Item { } public void ensureNotInSubtree(CompositeItem item) { - for (Iterator<Item> i = item.getItemIterator(); i.hasNext();) { - Item possibleCycle = i.next(); - - if (this == possibleCycle) { + for (Item i = this; i != null; i = i.getParent()) { + if (i == item) { throw new IllegalArgumentException("Cannot add " + item + " to " + this + " as it would create a cycle"); - } else if (possibleCycle instanceof CompositeItem) { - ensureNotInSubtree((CompositeItem) possibleCycle); } } } @@ -205,7 +201,7 @@ public abstract class CompositeItem extends Item { } /** Composite items should be parenthized when not on the top level */ - protected boolean shouldParenthize() { + protected boolean shouldParenthesize() { return getParent()!= null && ! (getParent() instanceof QueryTree); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java index 500c68cf4bd..43258db8963 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java @@ -20,7 +20,7 @@ public class EquivItem extends CompositeTaggableItem { public EquivItem() {} /** - * Creates an EQUIV with the given item as child. The new EQUIV will take connectivity, + * Creates an EQUIV with the given item as child. The new EQUIV will take connectivity, * significance and weight from the given item. * * @param item will be modified and added as a child @@ -48,7 +48,7 @@ public class EquivItem extends CompositeTaggableItem { // steal other item's weight: setWeight(item.getWeight()); - // we have now stolen all of the other item's unique id needs: + // we have now stolen all the other item's unique id needs: item.setHasUniqueID(false); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 92b321adb70..06fe07d3895 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -11,7 +11,6 @@ import com.yahoo.text.Utf8; import java.nio.ByteBuffer; import java.util.Objects; -import java.util.Optional; /** @@ -20,7 +19,7 @@ import java.util.Optional; * Items are in general mutable and not thread safe. * They can be deeply cloned by calling clone(). * Their identity is defined by their content - * (i.e the field value of two items decide if they are equal). + * (i.e. the field value of two items decide if they are equal). * * @author bratseth * @author havardpe @@ -60,7 +59,8 @@ public abstract class Item implements Cloneable { NEAREST_NEIGHBOR(26), GEO_LOCATION_TERM(27), TRUE(28), - FALSE(29); + FALSE(29), + MULTI_TERM(30); public final int code; @@ -233,36 +233,34 @@ public abstract class Item implements Cloneable { public abstract int encode(ByteBuffer buffer); protected void encodeThis(ByteBuffer buffer) { - int FEAT_SHIFT = 5; - int CODE_MASK = 0x1f; - int FEAT_MASK = 0xe0; - int FEAT_WEIGHT = 0x01; - int FEAT_UNIQUEID = 0x02; - int FEAT_FLAGS = 0x04; + byte CODE_MASK = 0b00011111; + byte FEAT_WEIGHT = 0b00100000; + byte FEAT_UNIQUEID = 0b01000000; + byte FEAT_FLAGS = -0b10000000; - int features = 0; + byte type = (byte) (getCode() & CODE_MASK); + if (type != getCode()) + throw new IllegalStateException("must increase number of bytes in serialization format for queries"); if (weight != DEFAULT_WEIGHT) { - features |= FEAT_WEIGHT; + type |= FEAT_WEIGHT; } if (hasUniqueID()) { - features |= FEAT_UNIQUEID; + type |= FEAT_UNIQUEID; } byte flags = getFlagsFeature(); if (flags != 0) { - features |= FEAT_FLAGS; + type |= FEAT_FLAGS; } - byte type = (byte)(((getCode() & CODE_MASK) - | ((features << FEAT_SHIFT) & FEAT_MASK)) & 0xff); buffer.put(type); - if ((features & FEAT_WEIGHT) != 0) { + if ((type & FEAT_WEIGHT) != 0) { IntegerCompressor.putCompressedNumber(weight, buffer); } - if ((features & FEAT_UNIQUEID) != 0) { + if ((type & FEAT_UNIQUEID) != 0) { IntegerCompressor.putCompressedPositiveNumber(uniqueID, buffer); } - if (flags != 0) { + if ((type & FEAT_FLAGS) != 0) { buffer.put(flags); } } @@ -297,7 +295,7 @@ public abstract class Item implements Cloneable { /** Utility method for turning a string into utf-8 bytes */ - protected static final byte[] getBytes(String string) { + protected static byte[] getBytes(String string) { return Utf8.toBytes(string); } public static void putString(String s, ByteBuffer buffer) { @@ -322,7 +320,7 @@ public abstract class Item implements Cloneable { public String toString() { StringBuilder buffer = new StringBuilder(); - if (shouldParenthize()) { + if (shouldParenthesize()) { buffer.append("("); } if (isFilter()) { @@ -330,7 +328,7 @@ public abstract class Item implements Cloneable { } appendHeadingString(buffer); appendBodyString(buffer); - if (shouldParenthize()) { + if (shouldParenthesize()) { buffer.append(")"); } @@ -343,10 +341,10 @@ public abstract class Item implements Cloneable { } /** - * Returns whether this item should be parethized when printed. + * Returns whether this item should be parenthesized when printed. * Default is false - no parentheses */ - protected boolean shouldParenthize() { + protected boolean shouldParenthesize() { return false; } @@ -376,7 +374,7 @@ public abstract class Item implements Cloneable { // note: connectedItem and connectedBacklink references are corrected in CompositeItem.clone() return clone; } catch (CloneNotSupportedException e) { - throw new RuntimeException("Someone made Item unclonable"); + throw new RuntimeException("Someone made Item uncloneable"); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java index fae282868f8..379dfd6bb30 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseItem.java @@ -202,7 +202,7 @@ public class PhraseItem extends CompositeIndexedItem { /** Returns false, no parenthezes for phrases */ @Override - protected boolean shouldParenthize() { + protected boolean shouldParenthesize() { return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 16e22f6d482..2a97970ac4a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -148,7 +148,7 @@ public class PhraseSegmentItem extends IndexedSegmentItem { /** Returns false, no parenthezes for phrases */ @Override - protected boolean shouldParenthize() { + protected boolean shouldParenthesize() { return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java index f23f2088f1d..eb23570ec90 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TaggableItem.java @@ -18,10 +18,10 @@ public interface TaggableItem { * This is used to influence ranking features taking proximity into account: nativeRank and a subset of the * fieldMatch features. * <p> - * By default consecutive query terms are 'somewhat' connected, meaning ranking features will be better in documents + * By default consecutive query terms are 'somewhat' connected, meaning ranking features will score higher in documents * where the terms are found close to each other. This effect can be increased or decreased by manipulating the * connectivity value. Typical use is to increase the connectivity between terms in the query that we believe are - * semantically connected. E.g in the query 'new york hotel', it is a good idea to increase the connectivity between + * semantically connected. E.g., in the query 'new york hotel', it is a good idea to increase the connectivity between * "new" and "york" to ensure that a document containing "List of hotels in New York" is ranked above one containing * "List of new hotels in York". * @@ -36,8 +36,8 @@ public interface TaggableItem { /** * Used for setting explicit term significance (in the tf/idf sense) to a single term or phrase, * relative to the rest of the query. - * This influences ranking features which take term significance into account and overrides the default - * partial corpus based term significance computation happening in the backend. + * This influences ranking features which take term significance into account, and overrides the default + * partial corpus based term significance computation in the backend. */ void setSignificance(double significance); boolean hasExplicitSignificance(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java index a988753d699..a5903f24cbd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java @@ -10,6 +10,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Objects; +import static java.util.Objects.requireNonNullElse; + /** * A term which contains a weighted set. * @@ -30,20 +32,12 @@ public class WeightedSetItem extends SimpleTaggableItem { /** Creates an empty weighted set; note you must provide an index name up front */ public WeightedSetItem(String indexName) { - if (indexName == null) { - this.indexName = ""; - } else { - this.indexName = indexName; - } + this.indexName = requireNonNullElse(indexName, ""); set = new CopyOnWriteHashMap<>(1000); } public WeightedSetItem(String indexName, Map<Object, Integer> map) { - if (indexName == null) { - this.indexName = ""; - } else { - this.indexName = indexName; - } + this.indexName = requireNonNullElse(indexName, ""); set = new CopyOnWriteHashMap<>(map); } @@ -96,11 +90,7 @@ public class WeightedSetItem extends SimpleTaggableItem { @Override public void setIndexName(String index) { - if (index == null) { - this.indexName = ""; - } else { - this.indexName = index; - } + this.indexName = requireNonNullElse(index, ""); } public String getIndexName() { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java index f4603e5ea28..0de4e36eae5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping; +import com.yahoo.api.annotations.Beta; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; @@ -37,6 +38,8 @@ public class GroupingQueryParser extends Searcher { public static final CompoundName PARAM_CONTINUE = new CompoundName("continue"); public static final CompoundName PARAM_REQUEST = new CompoundName(Select.SELECT); public static final CompoundName PARAM_TIMEZONE = new CompoundName("timezone"); + @Beta public static final CompoundName PARAM_DEFAULT_MAX_HITS = new CompoundName("grouping.defaultMaxHits"); + @Beta public static final CompoundName PARAM_DEFAULT_MAX_GROUPS = new CompoundName("grouping.defaultMaxGroups"); private static final ThreadLocal<ZoneCache> zoneCache = new ThreadLocal<>(); @Override @@ -52,6 +55,8 @@ public class GroupingQueryParser extends Searcher { grpRequest.setRootOperation(op); grpRequest.setTimeZone(zone); grpRequest.continuations().addAll(continuations); + grpRequest.setDefaultMaxGroups(query.properties().getInteger(PARAM_DEFAULT_MAX_GROUPS, -1)); + grpRequest.setDefaultMaxHits(query.properties().getInteger(PARAM_DEFAULT_MAX_HITS, -1)); } return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 6b21b2c6b5c..0c163aaacae 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping; +import com.yahoo.api.annotations.Beta; import com.yahoo.net.URI; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -13,6 +14,7 @@ import com.yahoo.search.result.Hit; import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; import java.util.TimeZone; /** @@ -30,6 +32,8 @@ public class GroupingRequest { private final List<Continuation> continuations = new ArrayList<>(); private GroupingOperation root; private TimeZone timeZone; + private int defaultMaxHits = -1; + private int defaultMaxGroups = -1; private GroupingRequest(Select parent) { this.parent = parent; @@ -38,16 +42,20 @@ public class GroupingRequest { private GroupingRequest(Select parent, List<Continuation> continuations, GroupingOperation root, - TimeZone timeZone) { + TimeZone timeZone, + int defaultMaxHits, + int defaultMaxGroups) { this.parent = parent; continuations.forEach(item -> this.continuations.add(item.copy())); this.root = root != null ? root.copy(null) : null; this.timeZone = timeZone; + this.defaultMaxHits = defaultMaxHits; + this.defaultMaxGroups = defaultMaxGroups; } /** Returns a deep copy of this */ public GroupingRequest copy(Select parentOfCopy) { - return new GroupingRequest(parentOfCopy, continuations, root, timeZone); + return new GroupingRequest(parentOfCopy, continuations, root, timeZone, defaultMaxHits, defaultMaxGroups); } /** @@ -131,6 +139,20 @@ public class GroupingRequest { return continuations; } + @Beta + public OptionalInt defaultMaxHits() { + return defaultMaxHits >= 0 ? OptionalInt.of(defaultMaxHits) : OptionalInt.empty(); + } + + @Beta public void setDefaultMaxHits(int v) { this.defaultMaxHits = v; } + + @Beta + public OptionalInt defaultMaxGroups() { + return defaultMaxGroups >= 0 ? OptionalInt.of(defaultMaxGroups) : OptionalInt.empty(); + } + + @Beta public void setDefaultMaxGroups(int v) { this.defaultMaxGroups = v; } + /** * Creates a new grouping request and adds it to the query.getSelect().getGrouping() list * diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 3e707b0cd38..c09502110b1 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -150,6 +150,8 @@ public class GroupingExecutor extends Searcher { builder.setDefaultSummaryName(query.getPresentation().getSummary()); builder.setTimeZone(req.getTimeZone()); builder.addContinuations(req.continuations()); + req.defaultMaxGroups().ifPresent(builder::setDefaultMaxGroups); + req.defaultMaxHits().ifPresent(builder::setDefaultMaxHits); builder.build(); RequestContext ctx = new RequestContext(req, builder.getTransform()); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java index e636c0d63a6..5ee6f7bc604 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/RequestBuilder.java @@ -27,6 +27,8 @@ class RequestBuilder { private final GroupingTransform transform; private GroupingOperation root; private int tag = 0; + private int defaultMaxHits = -1; + private int defaultMaxGroups = -1; /** * Constructs a new instance of this class. @@ -138,6 +140,10 @@ class RequestBuilder { return this; } + public RequestBuilder setDefaultMaxGroups(int v) { this.defaultMaxGroups = v; return this; } + + public RequestBuilder setDefaultMaxHits(int v) { this.defaultMaxHits = v; return this; } + private void processRequestNode(BuildFrame frame) { int level = frame.astNode.getLevel(); if (level > 2) { @@ -173,10 +179,16 @@ class RequestBuilder { grpLevel.setPrecision(frame.state.precision + offset); frame.state.precision = null; } + int max = -1; if (frame.state.max != null) { - transform.putMax(tag, frame.state.max, "group list"); - grpLevel.setMaxGroups(LOOKAHEAD + frame.state.max + offset); + max = frame.state.max; frame.state.max = null; + } else if (defaultMaxGroups >= 0) { + max = defaultMaxGroups; + } + if (max >= 0) { + transform.putMax(tag, max, "group list"); + grpLevel.setMaxGroups(LOOKAHEAD + max + offset); } frame.grouping.getLevels().add(grpLevel); } @@ -285,11 +297,17 @@ class RequestBuilder { throw new UnsupportedOperationException("Can not label expression '" + exp + "'."); } HitsAggregationResult hits = (HitsAggregationResult)result; + int max = -1; if (frame.state.max != null) { - transform.putMax(tag, frame.state.max, "hit list"); - int offset = transform.getOffset(tag); - hits.setMaxHits(LOOKAHEAD + frame.state.max + offset); + max = frame.state.max; frame.state.max = null; + } else if (defaultMaxHits >= 0) { + max = defaultMaxHits; + } + if (max >= 0) { + transform.putMax(tag, max, "hit list"); + int offset = transform.getOffset(tag); + hits.setMaxHits(LOOKAHEAD + max + offset); } transform.putLabel(group.getTag(), tag, frame.state.label, "hit list"); } else { diff --git a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java index ba225722fd0..418d7121769 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/ItemsCommonStuffTestCase.java @@ -212,7 +212,7 @@ public class ItemsCommonStuffTestCase { w.setConnectivity(v, 1.0); String expected = "puppy"; String expected2 = "kvalp"; - EquivItem e = new EquivItem(w, Arrays.asList(new String[] { expected, expected2 })); + EquivItem e = new EquivItem(w, Arrays.asList(expected, expected2)); assertEquals(1.0, e.getConnectivity(), 1e-9); assertSame(v, e.getConnectedItem()); assertEquals(expected, ((WordItem) e.getItem(1)).getWord()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java index 4397831af6e..29d9f7802c1 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -698,6 +698,36 @@ public class RequestBuilderTestCase { assertOutput(test); } + @Test + public void require_that_default_max_values_from_request_builder_restricts_max_groups_and_hits() { + int defaultMaxHits = 19; + int defaultMaxGroups = 7; + RequestBuilder builder = new RequestBuilder(0) + .setDefaultMaxGroups(defaultMaxGroups) + .setDefaultMaxHits(defaultMaxHits) + .setRootOperation(GroupingOperation.fromString("all(group(foo)each(each(output(summary()))))")); + builder.build(); + List<Grouping> requests = builder.getRequestList(); + assertEquals(defaultMaxGroups + 1, requests.get(0).getLevels().get(0).getMaxGroups()); + HitsAggregationResult hitsAggregation = + (HitsAggregationResult)requests.get(0).getLevels().get(0).getGroupPrototype().getAggregationResults().get(0); + assertEquals(defaultMaxHits + 1, hitsAggregation.getMaxHits()); + } + + @Test + public void require_that_default_max_values_from_request_builder_restricts_respects_explicit_max() { + RequestBuilder builder = new RequestBuilder(0) + .setDefaultMaxGroups(7) + .setDefaultMaxHits(19) + .setRootOperation(GroupingOperation.fromString("all(group(foo)max(11)each(max(21)each(output(summary()))))")); + builder.build(); + List<Grouping> requests = builder.getRequestList(); + assertEquals(12, requests.get(0).getLevels().get(0).getMaxGroups()); + HitsAggregationResult hitsAggregation = + (HitsAggregationResult)requests.get(0).getLevels().get(0).getGroupPrototype().getAggregationResults().get(0); + assertEquals(22, hitsAggregation.getMaxHits()); + } + private static CompositeContinuation newComposite(EncodableContinuation... conts) { CompositeContinuation ret = new CompositeContinuation(); for (EncodableContinuation cont : conts) { |