diff options
Diffstat (limited to 'container-search/src/main')
36 files changed, 487 insertions, 187 deletions
diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index f565315b775..d0d2cd3a442 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -14,11 +14,14 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.JsonRenderer; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.EventStream; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.Utf8; +import java.io.ByteArrayOutputStream; import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.function.Function; @@ -33,11 +36,15 @@ import java.util.stream.Collectors; @Beta public class LLMSearcher extends Searcher { - private static Logger log = Logger.getLogger(LLMSearcher.class.getName()); + private static final Logger log = Logger.getLogger(LLMSearcher.class.getName()); private static final String API_KEY_HEADER = "X-LLM-API-KEY"; private static final String STREAM_PROPERTY = "stream"; private static final String PROMPT_PROPERTY = "prompt"; + private static final String INCLUDE_PROMPT_IN_RESULT = "includePrompt"; + private static final String INCLUDE_HITS_IN_RESULT = "includeHits"; + + private final JsonRenderer jsonRenderer; private final String propertyPrefix; private final boolean stream; @@ -50,11 +57,13 @@ public class LLMSearcher extends Searcher { this.languageModelId = config.providerId(); this.languageModel = findLanguageModel(languageModelId, languageModels); this.propertyPrefix = config.propertyPrefix(); + + this.jsonRenderer = new JsonRenderer(); } @Override public Result search(Query query, Execution execution) { - return complete(query, StringPrompt.from(getPrompt(query))); + return complete(query, StringPrompt.from(getPrompt(query)), null, execution); } private LanguageModel findLanguageModel(String providerId, ComponentRegistry<LanguageModel> languageModels) @@ -81,30 +90,37 @@ public class LLMSearcher extends Searcher { return languageModel; } - protected Result complete(Query query, Prompt prompt) { + protected Result complete(Query query, Prompt prompt, Result result, Execution execution) { var options = new InferenceParameters(getApiKeyHeader(query), s -> lookupProperty(s, query)); var stream = lookupPropertyBool(STREAM_PROPERTY, query, this.stream); // query value overwrites config try { - return stream ? completeAsync(query, prompt, options) : completeSync(query, prompt, options); + if (stream) { + return completeAsync(query, prompt, options, result, execution); + } + return completeSync(query, prompt, options, result, execution); } catch (RejectedExecutionException e) { return new Result(query, new ErrorMessage(429, e.getMessage())); } } private boolean shouldAddPrompt(Query query) { - return query.getTrace().getLevel() >= 1; + var includePrompt = lookupPropertyBool(INCLUDE_PROMPT_IN_RESULT, query, false); + return query.getTrace().getLevel() >= 1 || includePrompt; } private boolean shouldAddTokenStats(Query query) { return query.getTrace().getLevel() >= 1; } - private Result completeAsync(Query query, Prompt prompt, InferenceParameters options) { + private Result completeAsync(Query query, Prompt prompt, InferenceParameters options, Result result, Execution execution) { final EventStream eventStream = new EventStream(); if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } + if (shouldAddHits(query) && result != null) { + eventStream.add(renderHits(result, execution), "hits"); + } final TokenStats tokenStats = new TokenStats(); languageModel.completeAsync(prompt, options, completion -> { @@ -143,12 +159,15 @@ public class LLMSearcher extends Searcher { eventStream.error(languageModelId, new ErrorMessage(errorCode, exception.getMessage())); } - private Result completeSync(Query query, Prompt prompt, InferenceParameters options) { + private Result completeSync(Query query, Prompt prompt, InferenceParameters options, Result result, Execution execution) { EventStream eventStream = new EventStream(); if (shouldAddPrompt(query)) { eventStream.add(prompt.asString(), "prompt"); } + if (shouldAddHits(query) && result != null) { + eventStream.add(renderHits(result, execution), "hits"); + } List<Completion> completions = languageModel.complete(prompt, options); eventStream.add(completions.get(0).text(), "completion"); @@ -200,9 +219,21 @@ public class LLMSearcher extends Searcher { return lookupPropertyWithOrWithoutPrefix(API_KEY_HEADER, p -> query.getHttpRequest().getHeader(p)); } + private boolean shouldAddHits(Query query) { + return lookupPropertyBool(INCLUDE_HITS_IN_RESULT, query, false); + } + + private String renderHits(Result results, Execution execution) { + var bs = new ByteArrayOutputStream(); + var renderer = jsonRenderer.clone(); + renderer.init(); + renderer.renderResponse(bs, results, execution, null).join(); // wait for renderer to complete + return Utf8.toString(bs.toByteArray()); + } + private static class TokenStats { - private long start; + private final long start; private long timeToFirstToken; private long timeToLastToken; private long tokens = 0; diff --git a/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java index cba153d881d..cdf57922bce 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/RAGSearcher.java @@ -37,7 +37,7 @@ public class RAGSearcher extends LLMSearcher { public Result search(Query query, Execution execution) { Result result = execution.search(query); execution.fill(result); - return complete(query, buildPrompt(query, result)); + return complete(query, buildPrompt(query, result), result, execution); } protected Prompt buildPrompt(Query query, Result result) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java index 6a980c37f4e..b7d7b3d9ed5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -1,14 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Class representing a byte field in the result set - * - */ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; /** + * Class representing a byte field in the result set + * * @author bratseth */ public class BoolField extends DocsumField { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 7f672582da3..02d970bf7c8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -28,8 +28,7 @@ public class DocsumDefinition { this.fields = documentSummary.fields().values() .stream() .map(field -> DocsumField.create(field.name(), field.type().asString())) - .collect(Collectors.toUnmodifiableMap(field -> field.getName(), - field -> field)); + .collect(Collectors.toUnmodifiableMap(DocsumField::getName, field -> field)); } public String name() { return name; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java index a463fb9d0e6..47a4530b9d1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackend.java @@ -311,10 +311,10 @@ public abstract class VespaBackend { if (query.getTrace().isTraceable(level + 1) && query.getTrace().getQuery()) { query.trace("Current state of query tree: " + new TextualQueryRepresentation(query.getModel().getQueryTree().getRoot()), - false, level+1); + false, level + 1); } if (query.getTrace().isTraceable(level + 2) && query.getTrace().getQuery()) { - query.trace("YQL+ representation: " + query.yqlRepresentation(), level+2); + query.trace("YQL+ representation: " + query.yqlRepresentation(), level + 2); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index de1d444434a..8325101b4ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -210,7 +210,7 @@ public abstract class CompositeItem extends Item { public CompositeItem clone() { CompositeItem copy = (CompositeItem) super.clone(); - copy.subitems = new java.util.ArrayList<>(subitems.size()); + copy.subitems = new ArrayList<>(subitems.size()); for (Item subItem : subitems) { Item subItemCopy = subItem.clone(); subItemCopy.setParent(copy); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java index 27213000e3a..25abf465199 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java @@ -5,13 +5,15 @@ import java.util.Objects; import static java.util.Objects.requireNonNullElse; -/* +/** * Abstract class representing an IN operator. * * @author toregge */ public abstract class InItem extends Item { + private String indexName; + public InItem(String indexName) { this.indexName = requireNonNullElse(indexName, ""); } @@ -20,6 +22,7 @@ public abstract class InItem extends Item { public void setIndexName(String index) { this.indexName = requireNonNullElse(index, ""); } + public String getIndexName() { return indexName; } @@ -43,4 +46,4 @@ public abstract class InItem extends Item { return Objects.hash(super.hashCode(), indexName); } -}; +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java index 2c6d69d3fac..9763e64b83c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/IntItem.java @@ -57,6 +57,10 @@ public class IntItem extends TermItem { expression = toExpression(from, to, 0); } + public IntItem(Long expression) { + this(expression, ""); + } + public IntItem(String expression) { this(expression, ""); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java index 9333173d898..e981e251064 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; import java.util.Collection; @@ -9,13 +10,14 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -/* +/** * Class representing an IN operator with a set of 64-bit * integer values. * * @author toregge */ public class NumericInItem extends InItem { + private Set<Long> tokens; public NumericInItem(String indexName) { @@ -23,6 +25,11 @@ public class NumericInItem extends InItem { tokens = new HashSet<>(1000); } + public NumericInItem(String indexName, Set<Long> tokens) { + super(indexName); + this.tokens = new HashSet<>(tokens); + } + @Override public Item.ItemType getItemType() { return Item.ItemType.NUMERIC_IN; @@ -73,6 +80,13 @@ public class NumericInItem extends InItem { public Collection<Long> getTokens() { return Set.copyOf(tokens); } @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + for (Long token : tokens) + discloser.addChild(new IntItem(token)); + } + + @Override public boolean equals(Object o) { if (o == this) return true; if ( ! super.equals(o)) return false; @@ -86,4 +100,11 @@ public class NumericInItem extends InItem { return Objects.hash(super.hashCode(), tokens); } + @Override + public NumericInItem clone() { + NumericInItem clone = (NumericInItem)super.clone(); + clone.tokens = new HashSet<>(tokens); + return clone; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java index 4473010082e..caa066eddd3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query; import com.yahoo.compress.IntegerCompressor; +import com.yahoo.prelude.query.textualrepresentation.Discloser; import java.nio.ByteBuffer; import java.util.Collection; @@ -9,12 +10,13 @@ import java.util.HashSet; import java.util.Objects; import java.util.Set; -/* +/** * Class representing an IN operator with a set of string values. * * @author toregge */ public class StringInItem extends InItem { + private Set<String> tokens; public StringInItem(String indexName) { @@ -22,6 +24,11 @@ public class StringInItem extends InItem { tokens = new HashSet<>(1000); } + public StringInItem(String indexName, Set<String> tokens) { + super(indexName); + this.tokens = new HashSet<>(tokens); + } + @Override public ItemType getItemType() { return ItemType.STRING_IN; @@ -72,14 +79,21 @@ public class StringInItem extends InItem { tokens.remove(token); } - public Collection<String> getTokens() { return Set.copyOf(tokens); } + public Collection<String> getTokens() {return Set.copyOf(tokens);} + + @Override + public void disclose(Discloser discloser) { + super.disclose(discloser); + for (String token : tokens) + discloser.addChild(new WordItem(token)); + } @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! super.equals(o)) return false; - var other = (StringInItem)o; - if ( ! Objects.equals(this.tokens, other.tokens)) return false; + if (!super.equals(o)) return false; + var other = (StringInItem) o; + if (!Objects.equals(this.tokens, other.tokens)) return false; return true; } @@ -88,4 +102,11 @@ public class StringInItem extends InItem { return Objects.hash(super.hashCode(), tokens); } + @Override + public StringInItem clone() { + StringInItem clone = (StringInItem) super.clone(); + clone.tokens = new HashSet<>(tokens); + return clone; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 7d258f71ebd..5e9c3dcc6ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -335,7 +335,7 @@ public abstract class AbstractParser implements CustomParser { /** * Segments a token * - * @param indexName the index name which preceeded this token, or null if none + * @param indexName the index name which preceded this token, or null if none * @param token the token to segment * @param quoted whether this segment is within quoted text * @return the resulting item diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index ea0cd2312a6..d3e6241a6e5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -134,10 +134,8 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); - return not; - } else { - return not; } + return not; } if (topLevelItem != null) { return topLevelItem; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index f27e9bbc281..0ea9cefc63e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -109,14 +109,17 @@ public class StemmingSearcher extends Searcher { private Item replaceTerms(Query q, IndexFacts.Session indexFacts) { Language language = q.getModel().getParsingLanguage(); - if (language == Language.UNKNOWN) return q.getModel().getQueryTree().getRoot(); + if (language == Language.UNKNOWN) { + q.trace("Language is unknown, not stemming", 3); + return q.getModel().getQueryTree().getRoot(); + } StemContext context = new StemContext(); context.isCJK = language.isCjk(); context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); - q.trace("Stemming with language="+language, 3); + q.trace("Stemming with language " + language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 4ec3fa358d2..8e0897b866f 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -384,9 +384,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // We need special handling for "select" because it can be both the prefix of the nested JSON select // parameters, and a plain select expression. The latter will be disallowed by query profile types // since they contain the former. - String select = requestMap.get(Select.SELECT); + Object select = requestMap.get(Select.SELECT); + if (select == null) + select = queryProfile.get(Select.SELECT, requestMap); if (select != null) - properties().set(Select.SELECT, select); + properties().set(Select.SELECT, select.toString()); } else { // bypass these complications if there is no query profile to get values from and validate against properties(). @@ -605,7 +607,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setHits(int hits) { if (hits < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'hits' must be a positive number, not " + hits); this.hits = hits; } @@ -614,12 +616,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setOffset(int offset) { if (offset < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'offset' must be a positive number, not " + offset); this.offset = offset; } /** Convenience method to set both the offset and the number of hits to return */ - public void setWindow(int offset,int hits) { + public void setWindow(int offset, int hits) { setOffset(offset); setHits(hits); } @@ -906,6 +908,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the select to be used for this query, never null */ public Select getSelect() { return select; } + /** Sets the select (grouping) parameter from a string. */ + public void setSelect(String groupingString) { select.setGroupingExpressionString(groupingString); } + /** Returns the ranking to be used for this query, never null */ public Ranking getRanking() { return ranking; } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index d7fad148c8c..bfcf0af325d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -79,7 +79,7 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! cluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { + if ( ! cluster.isPartialGroupCoverageSufficient(group.hasSufficientCoverage(), success) && !acceptIncompleteCoverage) { return Optional.empty(); } if (invokers.isEmpty()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 965ce4aeb94..c7af37b3a26 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -23,7 +23,7 @@ public class Group { // Using volatile to ensure visibility for reader. // All updates are done in a single writer thread - private volatile boolean hasSufficientCoverage = true; + private volatile boolean hasSufficientCoverage = false; private volatile boolean hasFullCoverage = true; private volatile long activeDocuments = 0; private volatile long targetActiveDocuments = 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 56545a32831..8f83d8ef5ce 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -226,17 +226,20 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), group.activeDocuments()); - trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), group.activeDocuments(), group.activeDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments(), group.activeDocuments()); } private void pingIterationCompletedMultipleGroups(SearchGroupsImpl groups) { groups.groups().forEach(Group::aggregateNodeValues); - long medianDocuments = groups.medianDocumentsPerGroup(); + long medianDocuments = groups.medianDocumentCount(); + long maxDocuments = groups.maxDocumentCount(); for (Group group : groups.groups()) { - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), medianDocuments); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), medianDocuments, maxDocuments); updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments, maxDocuments); } } @@ -261,7 +264,7 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments, long maxDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.fullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { @@ -278,7 +281,7 @@ public class SearchCluster implements NodeManager<Node> { unresponsive.append('\n').append(node); } String message = "Cluster " + clusterId + ": " + group + " has reduced coverage: " + - "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Active documents: " + group.activeDocuments() + "/" + maxDocuments + ", " + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java index 85063b8ef57..0bb694f610e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java @@ -13,21 +13,30 @@ import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toSet; /** - * Simple interface for groups and their nodes in the content cluster + * Simple interface for groups and their nodes in the content cluster. + * * @author baldersheim */ public interface SearchGroups { + Group get(int id); + Set<Integer> keys(); + Collection<Group> groups(); + default boolean isEmpty() { return size() == 0; } + default Set<Node> nodes() { return groups().stream().flatMap(group -> group.nodes().stream()) .sorted(comparingInt(Node::key)) .collect(toCollection(LinkedHashSet::new)); } + int size(); - boolean isPartialGroupCoverageSufficient(Collection<Node> nodes); + + boolean isPartialGroupCoverageSufficient(boolean currentCoverageSufficient, Collection<Node> nodes); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java index c49a140804c..6528c5d2ae4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java @@ -7,14 +7,17 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +/** + * @author baldersheim + */ public class SearchGroupsImpl implements SearchGroups { private final Map<Integer, Group> groups; - private final double minActivedocsPercentage; + private final double minActiveDocsPercentage; - public SearchGroupsImpl(Map<Integer, Group> groups, double minActivedocsPercentage) { + public SearchGroupsImpl(Map<Integer, Group> groups, double minActiveDocsPercentage) { this.groups = Map.copyOf(groups); - this.minActivedocsPercentage = minActivedocsPercentage; + this.minActiveDocsPercentage = minActiveDocsPercentage; } @Override public Group get(int id) { return groups.get(id); } @@ -23,23 +26,38 @@ public class SearchGroupsImpl implements SearchGroups { @Override public int size() { return groups.size(); } @Override - public boolean isPartialGroupCoverageSufficient(Collection<Node> nodes) { - if (size() == 1) - return true; - long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); - return isGroupCoverageSufficient(activeDocuments, medianDocumentsPerGroup()); + public boolean isPartialGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, Collection<Node> nodes) { + if (size() == 1) return true; + long groupDocumentCount = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(currentIsGroupCoverageSufficient, + groupDocumentCount, medianDocumentCount(), maxDocumentCount()); } - public boolean isGroupCoverageSufficient(long activeDocuments, long medianDocuments) { - if (medianDocuments <= 0) return true; - double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; - return documentCoverage >= minActivedocsPercentage; + public boolean isGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, + long groupDocumentCount, long medianDocumentCount, long maxDocumentCount) { + if (medianDocumentCount <= 0) return true; + if (currentIsGroupCoverageSufficient) { + // To take a group *out of* rotation, require that it has less active documents than the median. + // This avoids scenarios where incorrect accounting in a single group takes all other groups offline. + double documentCoverage = 100.0 * (double) groupDocumentCount / medianDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } + else { + // to put a group *in* rotation, require that it has as many documents as the largest group, + // to avoid taking groups in too early when the majority of the groups have just been added. + double documentCoverage = 100.0 * (double) groupDocumentCount / maxDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } } - public long medianDocumentsPerGroup() { + public long medianDocumentCount() { if (isEmpty()) return 0; double[] activeDocuments = groups().stream().mapToDouble(Group::activeDocuments).toArray(); return (long) Quantiles.median().computeInPlace(activeDocuments); } + public long maxDocumentCount() { + return (long)groups().stream().mapToDouble(Group::activeDocuments).max().orElse(0); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index fba9064298c..73360a905d2 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -218,7 +218,7 @@ public class GroupingExecutor extends Searcher { if (lastPass > 0) { baseRoot = origRoot.clone(); } - if (query.getTrace().isTraceable(3) && query.getGroupingSessionCache()) { + if (query.getTrace().isTraceable(3)) { query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index 01167be6b8b..fdedbdc2fd9 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -64,8 +64,8 @@ class Json2SingleLevelMap { } void parse(Map<String, String> map, String parent) throws IOException { - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parent + parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parent + parser.currentName(); JsonToken token = parser.nextToken(); if ((token == JsonToken.VALUE_STRING) || (token == JsonToken.VALUE_NUMBER_FLOAT) || @@ -89,9 +89,9 @@ class Json2SingleLevelMap { } private String skipChildren(JsonParser parser, byte [] input) throws IOException { - JsonLocation start = parser.getCurrentLocation(); + JsonLocation start = parser.currentLocation(); parser.skipChildren(); - JsonLocation end = parser.getCurrentLocation(); + JsonLocation end = parser.currentLocation(); int offset = (int)start.getByteOffset() - 1; return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index b1dd5624d18..a559846d5fe 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -14,6 +14,7 @@ import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.ranking.Matching; import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.search.query.ranking.RankProperties; +import com.yahoo.search.query.ranking.SecondPhase; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.result.ErrorMessage; @@ -45,6 +46,7 @@ public class Ranking implements Cloneable { public static final String KEEPRANKCOUNT = "keepRankCount"; public static final String RANKSCOREDROPLIMIT = "rankScoreDropLimit"; public static final String MATCH_PHASE = "matchPhase"; + public static final String SECOND_PHASE = "secondPhase"; public static final String GLOBAL_PHASE = "globalPhase"; public static final String DIVERSITY = "diversity"; public static final String SOFTTIMEOUT = "softtimeout"; @@ -69,6 +71,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(RANKSCOREDROPLIMIT, "double")); argumentType.addField(new FieldDescription(GLOBAL_PHASE, new QueryProfileFieldType(GlobalPhase.getArgumentType()))); argumentType.addField(new FieldDescription(MATCH_PHASE, new QueryProfileFieldType(MatchPhase.getArgumentType()), "matchPhase")); + argumentType.addField(new FieldDescription(SECOND_PHASE, new QueryProfileFieldType(SecondPhase.getArgumentType()))); argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); argumentType.addField(new FieldDescription(MATCHING, new QueryProfileFieldType(Matching.getArgumentType()))); @@ -107,12 +110,16 @@ public class Ranking implements Cloneable { private MatchPhase matchPhase = new MatchPhase(); + private SecondPhase secondPhase = new SecondPhase(); + private GlobalPhase globalPhase = new GlobalPhase(); private Matching matching = new Matching(); private SoftTimeout softTimeout = new SoftTimeout(); + private boolean useSignificance = false; + public Ranking(Query parent) { this.parent = parent; this.rankFeatures = new RankFeatures(this); @@ -217,9 +224,20 @@ public class Ranking implements Cloneable { /** Returns whether rank features should be dumped with the result of this query, default false */ public boolean getListFeatures() { return listFeatures; } + /** Set whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public void setUseSignificance(boolean useSignificance) { this.useSignificance = useSignificance; } + + /** Returns whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public boolean getUseSignificance() { return useSignificance; } + /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } + /** Return the second-phase rank settings of this. This is never null. */ + public SecondPhase getSecondPhase() { return secondPhase; } + /** Returns the global-phase rank settings of this. This is never null. */ public GlobalPhase getGlobalPhase() { return globalPhase; } @@ -256,6 +274,7 @@ public class Ranking implements Cloneable { public void prepare() { rankFeatures.prepare(rankProperties); matchPhase.prepare(rankProperties); + secondPhase.prepare(rankProperties); matching.prepare(rankProperties); softTimeout.prepare(rankProperties); prepareNow(freshness); diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 6735a6bd050..38ef7b8f190 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -115,9 +115,7 @@ public class Select implements Cloneable { public String getGroupingExpressionString() { return groupingExpressionString; } /** Returns the grouping in the query */ - public String getGroupingString(){ - return grouping; - } + public String getGroupingString() { return grouping; } /** * Returns the query's {@link GroupingRequest} as a mutable list. Changing this directly changes the grouping diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index c90612425fa..90d5e04d2b6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -256,26 +256,18 @@ public class SelectParser implements Parser { } private Item buildFunctionCall(String key, Inspector value) { - switch (key) { - case WAND: - return buildWand(key, value); - case WEIGHTED_SET: - return buildWeightedSet(key, value); - case DOT_PRODUCT: - return buildDotProduct(key, value); - case GEO_LOCATION: - return buildGeoLocation(key, value); - case NEAREST_NEIGHBOR: - return buildNearestNeighbor(key, value); - case PREDICATE: - return buildPredicate(key, value); - case RANK: - return buildRank(key, value); - case WEAK_AND: - return buildWeakAnd(key, value); - default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); - } + return switch (key) { + case WAND -> buildWand(key, value); + case WEIGHTED_SET -> buildWeightedSet(key, value); + case DOT_PRODUCT -> buildDotProduct(key, value); + case GEO_LOCATION -> buildGeoLocation(key, value); + case NEAREST_NEIGHBOR -> buildNearestNeighbor(key, value); + case PREDICATE -> buildPredicate(key, value); + case RANK -> buildRank(key, value); + case WEAK_AND -> buildWeakAnd(key, value); + default -> + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + }; } private void addItemsFromInspector(CompositeItem item, Inspector inspector){ @@ -312,15 +304,11 @@ public class SelectParser implements Parser { private HashMap<Integer, Inspector> childMap(Inspector inspector) { HashMap<Integer, Inspector> children = new HashMap<>(); if (inspector.type() == ARRAY){ - inspector.traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.traverse((ArrayTraverser) children::put); } else if (inspector.type() == OBJECT){ if (inspector.field("children").valid()){ - inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.field("children").traverse((ArrayTraverser) children::put); } } return children; @@ -336,9 +324,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation) { HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ - annotation.traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + annotation.traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -346,9 +332,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMap(Inspector inspector) { HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ - inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + inspector.field("attributes").traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -487,7 +471,6 @@ public class SelectParser implements Parser { return item; } - @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); @@ -576,8 +559,7 @@ public class SelectParser implements Parser { } }); } - if (out instanceof IntItem && annotations != null) { - IntItem number = (IntItem) out; + if (out instanceof IntItem number && annotations != null) { Integer hitLimit = getCappedRangeSearchParameter(annotations); if (hitLimit != null) { number.setHitLimit(hitLimit); @@ -631,12 +613,13 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("The first array element under 'equals' should be a field name string " + "but was " + children.get(0)); String field = children.get(0).asString(); - switch (children.get(1).type()) { - case BOOL: return new BoolItem(children.get(1).asBool(), field); - case LONG: return new IntItem(children.get(1).asLong(), field); - default: throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + - "or int value but was " + children.get(1)); - } + return switch (children.get(1).type()) { + case BOOL -> new BoolItem(children.get(1).asBool(), field); + case LONG -> new IntItem(children.get(1).asLong(), field); + default -> + throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + + "or int value but was " + children.get(1)); + }; } private Item buildRange(String key, Inspector value) { @@ -661,15 +644,15 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("Expected a numeric argument to range, but got the string '" + bound.asString() + "'"); } if (operator.equals("=")) { - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; equals[0] = true; } if (operator.equals(">=") || operator.equals(">")){ - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; } else if (operator.equals("<=") || operator.equals("<")){ - bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[1] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[1] = operator; } @@ -680,20 +663,13 @@ public class SelectParser implements Parser { } else if (operators[0] == null || operators[1] == null) { int index = (operators[0] == null) ? 1 : 0; - switch (operators[index]){ - case ">=": - range = buildGreaterThanOrEquals(field, bounds[index].toString()); - break; - case ">": - range = buildGreaterThan(field, bounds[index].toString()); - break; - case "<": - range = buildLessThan(field, bounds[index].toString()); - break; - case "<=": - range = buildLessThanOrEquals(field, bounds[index].toString()); - break; - } + range = switch (operators[index]) { + case ">=" -> buildGreaterThanOrEquals(field, bounds[index].toString()); + case ">" -> buildGreaterThan(field, bounds[index].toString()); + case "<" -> buildLessThan(field, bounds[index].toString()); + case "<=" -> buildLessThanOrEquals(field, bounds[index].toString()); + default -> range; + }; } else { range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<")); @@ -890,7 +866,7 @@ public class SelectParser implements Parser { String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; if (FUNCTION_CALLS.contains(key)) { return instantiateCompositeLeaf(field, key, value); - } else if ( ! possibleLeafFunctionName.equals("")){ + } else if (!possibleLeafFunctionName.isEmpty()){ return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); } else { return instantiateWordItem(field, key, value); @@ -898,24 +874,16 @@ public class SelectParser implements Parser { } private Item instantiateCompositeLeaf(String field, String key, Inspector value) { - switch (key) { - case SAME_ELEMENT: - return instantiateSameElementItem(field, key, value); - case PHRASE: - return instantiatePhraseItem(field, key, value); - case NEAR: - return instantiateNearItem(field, key, value); - case ONEAR: - return instantiateONearItem(field, key, value); - case EQUIV: - return instantiateEquivItem(field, key, value); - case FUZZY: - return instantiateFuzzyItem(field, key, value); - case ALTERNATIVES: - return instantiateWordAlternativesItem(field, key, value); - default: - throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); - } + return switch (key) { + case SAME_ELEMENT -> instantiateSameElementItem(field, key, value); + case PHRASE -> instantiatePhraseItem(field, key, value); + case NEAR -> instantiateNearItem(field, key, value); + case ONEAR -> instantiateONearItem(field, key, value); + case EQUIV -> instantiateEquivItem(field, key, value); + case FUZZY -> instantiateFuzzyItem(field, key, value); + case ALTERNATIVES -> instantiateWordAlternativesItem(field, key, value); + default -> throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); + }; } private Item instantiateWordItem(String field, String key, Inspector value) { @@ -944,8 +912,8 @@ public class SelectParser implements Parser { Preconditions.checkArgument((prefixMatch ? 1 : 0) + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, "Only one of prefix, substring and suffix can be set."); - final TaggableItem wordItem; + WordItem wordItem; if (exactMatch) { wordItem = new ExactStringItem(wordData, fromQuery); } else if (prefixMatch) { @@ -958,13 +926,11 @@ public class SelectParser implements Parser { wordItem = new WordItem(wordData, fromQuery); } - if (wordItem instanceof WordItem) { - prepareWord(field, value, (WordItem) wordItem); - } + prepareWord(field, value, wordItem); if (language != Language.ENGLISH) - ((Item)wordItem).setLanguage(language); + wordItem.setLanguage(language); - return (Item) leafStyleSettings(getAnnotations(value), wordItem); + return leafStyleSettings(getAnnotations(value), wordItem); } private Language decideParsingLanguage(Inspector value, String wordData) { @@ -974,9 +940,8 @@ public class SelectParser implements Parser { if (language != Language.UNKNOWN) return language; Optional<Language> explicitLanguage = query.getExplicitLanguage(); - if (explicitLanguage.isPresent()) return explicitLanguage.get(); + return explicitLanguage.orElse(Language.ENGLISH); - return Language.ENGLISH; } private void prepareWord(String field, Inspector value, WordItem wordItem) { @@ -1094,7 +1059,7 @@ public class SelectParser implements Parser { Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); if (distance != null) { - near.setDistance((int)distance); + near.setDistance(distance); } return near; } @@ -1120,7 +1085,8 @@ public class SelectParser implements Parser { private Item instantiateEquivItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size()); + Preconditions.checkArgument(children.size() >= 2, + "Expected 2 or more arguments, got %s.", children.size()); EquivItem equiv = new EquivItem(); equiv.setIndexName(field); @@ -1159,8 +1125,9 @@ public class SelectParser implements Parser { private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); - Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type()); + Preconditions.checkArgument(!children.isEmpty(), "Expected 1 or more arguments, got none"); + Preconditions.checkArgument(children.get(0).type() == OBJECT, + "Expected OBJECT, got %s.", children.get(0).type()); List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 8806854b9ce..29791712c9d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -101,6 +101,7 @@ public class QueryProperties extends Properties { map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.MINGROUPS), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getMinGroups(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setMinGroups(asLong(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.FACTOR), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffFactor(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffFactor(asDouble(value, 10.0)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.MATCH_PHASE, Ranking.DIVERSITY, Diversity.CUTOFF, Diversity.STRATEGY), GetterSetter.of(query -> query.getRanking().getMatchPhase().getDiversity().getCutoffStrategy(), (query, value) -> query.getRanking().getMatchPhase().getDiversity().setCutoffStrategy(asString(value, "loose")))); + map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.SECOND_PHASE, Ranking.RANKSCOREDROPLIMIT), GetterSetter.of(query -> query.getRanking().getSecondPhase().getRankScoreDropLimit(), (query, value) -> query.getRanking().getSecondPhase().setRankScoreDropLimit(asDouble(value, null)))); map.put(CompoundName.fromComponents(Ranking.RANKING, Ranking.GLOBAL_PHASE, Ranking.RERANKCOUNT), GetterSetter.of(query -> query.getRanking().getGlobalPhase().getRerankCount(), (query, value) -> query.getRanking().getGlobalPhase().setRerankCount(asInteger(value, null)))); diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java new file mode 100644 index 00000000000..0f6564d827f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SecondPhase.java @@ -0,0 +1,73 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.ranking; + +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +import java.util.Objects; + +/** + * The second-phase ranking settings of this query. + * + * @author toregge + */ +public class SecondPhase implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + + static { + argumentType = new QueryProfileType(Ranking.SECOND_PHASE); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(Ranking.RANKSCOREDROPLIMIT, FieldType.doubleType)); + argumentType.freeze(); + } + public static QueryProfileType getArgumentType() { return argumentType; } + + private Double rankScoreDropLimit = null; + + /** Sets the second phase rank-score-drop-limit that will be used, or null if not set */ + public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } + + /** Returns the second phase rank-score-drop-limit that will be used, or null if not set */ + public Double getRankScoreDropLimit() { return rankScoreDropLimit; } + + /** Internal operation - DO NOT USE */ + public void prepare(RankProperties rankProperties) { + if (rankScoreDropLimit == null) { + return; + } + rankProperties.put("vespa.hitcollector.secondphase.rankscoredroplimit", String.valueOf(rankScoreDropLimit)); + } + + @Override + public int hashCode() { + return Objects.hashCode(this.rankScoreDropLimit); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (o instanceof SecondPhase other) { + if ( ! Objects.equals(this.rankScoreDropLimit, other.rankScoreDropLimit)) return false; + return true; + } + return false; + } + + @Override + public SecondPhase clone() { + try { + SecondPhase clone = (SecondPhase)super.clone(); + clone.rankScoreDropLimit = this.rankScoreDropLimit; + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 7536e74042c..72a1a7d3430 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -12,6 +12,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.MinimalQueryInserter; import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Provides; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -19,10 +20,12 @@ import com.yahoo.yolean.chain.After; * * @author karowan */ +@Provides(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) @After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { + public static final String REPLACE_OR_WITH_WEAKAND = "replace-or-with-weakand"; static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace"); - static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); + public static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); @Override public Result search(Query query, Execution execution) { if (!query.properties().getBoolean(WEAKAND_REPLACE)) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java index 88a1e6c1485..ffbb63514f1 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/EventRenderer.java @@ -79,13 +79,16 @@ public class EventRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeRaw("event: " + event.type() + "\n"); } generator.writeRaw("data: "); - generator.writeStartObject(); - generator.writeStringField(event.type(), event.toString()); - generator.writeEndObject(); + if (event.type().equals("hits")) { + generator.writeRaw(event.toString()); + } else { + generator.writeStartObject(); + generator.writeStringField(event.type(), event.toString()); + generator.writeEndObject(); + } generator.writeRaw("\n\n"); generator.flush(); } - // Todo: support other types of data such as search results (hits), timing and trace } @Override diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index a5b8d328a7a..9583e9885e7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -36,6 +36,7 @@ public class RankProfile { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<String, InputType> inputs; // Assigned when this is added to a schema @@ -45,6 +46,7 @@ public class RankProfile { this.name = builder.name; this.hasSummaryFeatures = builder.hasSummaryFeatures; this.hasRankFeatures = builder.hasRankFeatures; + this.useSignificanceModel = builder.useSignificanceModel; this.inputs = Collections.unmodifiableMap(builder.inputs); } @@ -66,6 +68,9 @@ public class RankProfile { /** Returns true if this rank profile has rank features. */ public boolean hasRankFeatures() { return hasRankFeatures; } + /** Returns true if this rank profile should use significance models. */ + public boolean useSignificanceModel() { return useSignificanceModel; } + /** Returns the inputs explicitly declared in this rank profile. */ public Map<String, InputType> inputs() { return inputs; } @@ -76,13 +81,14 @@ public class RankProfile { if ( ! other.name.equals(this.name)) return false; if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; if ( other.hasRankFeatures != this.hasRankFeatures) return false; + if ( other.useSignificanceModel != this.useSignificanceModel) return false; if ( ! other.inputs.equals(this.inputs)) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); + return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, useSignificanceModel, inputs); } @Override @@ -95,6 +101,7 @@ public class RankProfile { private final String name; private boolean hasSummaryFeatures = true; private boolean hasRankFeatures = true; + private boolean useSignificanceModel = false; private final Map<String, InputType> inputs = new LinkedHashMap<>(); public Builder(String name) { @@ -116,6 +123,8 @@ public class RankProfile { return this; } + public Builder setUseSignificanceModel(boolean use) { this.useSignificanceModel = use; return this; } + public RankProfile build() { return new RankProfile(this); } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index d28c2db2b9e..77f27d3d411 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -22,9 +22,10 @@ class SchemaInfoConfigurer { Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); for (var profileConfig : schemaInfoConfig.rankprofile()) { - RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); - profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); - profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); + RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()) + .setHasSummaryFeatures(profileConfig.hasSummaryFeatures()) + .setHasRankFeatures(profileConfig.hasRankFeatures()) + .setUseSignificanceModel(profileConfig.significance().useModel()); for (var inputConfig : profileConfig.input()) profileBuilder.addInput(inputConfig.name(), RankProfile.InputType.fromSpec(inputConfig.type())); builder.add(profileBuilder.build()); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java index 84563818007..b7aeecdb7c8 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/AsyncExecution.java @@ -121,11 +121,7 @@ public class AsyncExecution { }, query); } - /** - * The future of this functions returns the original Result - * - * @see com.yahoo.search.searchchain.Execution - */ + /** Fills this result and returns the future where it is filled. */ public FutureResult fill(Result result, String summaryClass) { return getFutureResult(execution.context().executor(), () -> { execution.fill(result, summaryClass); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 6eb69c76afd..08526ecbaef 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -499,16 +499,6 @@ public class Execution extends com.yahoo.processing.execution.Execution { return (Result)super.process(query); } - @Override - protected void onInvoking(Request request, Processor processor) { - super.onInvoking(request,processor); - final int traceDependencies = 6; - Query query = (Query) request; - if (query.getTrace().getLevel() >= traceDependencies) { - query.trace(processor.getId() + " " + processor.getDependencies(), traceDependencies); - } - } - /** * The default response returned from this kind of execution when there are not further processors * - an empty Result diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index 69a1f8ec6cb..c03a74ea2c5 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -34,7 +34,8 @@ public class VespaSearchers { com.yahoo.prelude.searcher.PosSearcher.class, com.yahoo.prelude.semantics.SemanticSearcher.class, com.yahoo.search.grouping.GroupingQueryParser.class, - com.yahoo.search.querytransform.WeakAndReplacementSearcher.class); + com.yahoo.search.querytransform.WeakAndReplacementSearcher.class, + com.yahoo.search.searchers.OpportunisticWeakAndSearcher.class); public static final Collection<ChainedComponentModel> nativeSearcherModels; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java new file mode 100644 index 00000000000..2995b9e830e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java @@ -0,0 +1,85 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.searchers; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.WeakAndReplacementSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Will opportunistically replace the WeakAND with an AND as it is faster. + * If enough hits are returned all is good and we return. If not we fall back to the original query. + * + * @author baldersheim + */ +@Beta +@After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) +public class OpportunisticWeakAndSearcher extends Searcher { + private static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and"); + private static final CompoundName OPPORTUNISTIC_FACTOR = CompoundName.from("weakAnd.opportunistic.factor"); + + @Override + public Result search(Query query, Execution execution) { + if (!query.properties().getBoolean(OPPORTUNISTIC_AND)) { + return execution.search(query); + } + + Item originalRoot = query.getModel().getQueryTree().getRoot(); + int targetHits = (int)(targetHits(originalRoot) * query.properties().getDouble(OPPORTUNISTIC_FACTOR, 1.0)); + if (targetHits >= 0) { + query.getModel().getQueryTree().setRoot(weakAnd2AndRecurse(originalRoot.clone())); + query.trace("WeakAND(" + targetHits+ ") => AND", true, 2); + Result result = execution.search(query); + if (result.getTotalHitCount() >= targetHits) { + return result; + } + query.getModel().getQueryTree().setRoot(originalRoot); + query.trace("Fallback to WeakAND(" + targetHits+ ") as AND => " + result, true, 2); + return execution.search(query); + } + return execution.search(query); + } + + // returns targetHits for the first WeakAndItem found, -1 if none found. + static int targetHits(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return -1; + if (item instanceof WeakAndItem weakAndItem) return weakAndItem.getN(); + for (int i = 0; i < compositeItem.getItemCount(); i++) { + int targetHits = targetHits(compositeItem.getItem(i)); + if (targetHits >= 0) return targetHits; + } + return -1; + } + + static Item weakAnd2AndRecurse(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return item; + compositeItem = weakAnd2And(compositeItem); + for (int i = 0; i < compositeItem.getItemCount(); i++) { + Item subItem = compositeItem.getItem(i); + Item replacedItem = weakAnd2AndRecurse(subItem); + if (replacedItem != subItem) { + compositeItem.setItem(i, replacedItem); + } + } + return compositeItem; + } + + private static CompositeItem weakAnd2And(CompositeItem item) { + if (item instanceof WeakAndItem weakAndItem) { + AndItem andItem = new AndItem(); + andItem.setWeight(weakAndItem.getWeight()); + item.items().forEach(andItem::addItem); + return andItem; + } + return item; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index 0a42bf8a259..e3a559da8f9 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -14,33 +14,72 @@ import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; +import java.util.HashSet; import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; /** * Sets significance values on word items in the query tree. * - * @author MariusArhaug + * @author Marius Arhaug */ - @Provides(SignificanceSearcher.SIGNIFICANCE) @Before(STEMMING) public class SignificanceSearcher extends Searcher { public final static String SIGNIFICANCE = "Significance"; - private final SignificanceModelRegistry significanceModelRegistry; + private static final Logger log = Logger.getLogger(SignificanceSearcher.class.getName()); + + private final SignificanceModelRegistry significanceModelRegistry; + private final SchemaInfo schemaInfo; @Inject - public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry) { + public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry, SchemaInfo schemaInfo) { this.significanceModelRegistry = significanceModelRegistry; + this.schemaInfo = schemaInfo; } @Override public Result search(Query query, Execution execution) { + var rankProfileName = query.getRanking().getProfile(); + + // Determine significance setup per schema for the given rank profile + var perSchemaSetup = schemaInfo.newSession(query).schemas().stream() + .collect(Collectors.toMap(Schema::name, schema -> + // Fallback to disabled if the rank profile is not found in the schema + // This will result in a failure later (in a "backend searcher") anyway. + Optional.ofNullable(schema.rankProfiles().get(rankProfileName)) + .map(RankProfile::useSignificanceModel).orElse(false))); + var uniqueSetups = new HashSet<>(perSchemaSetup.values()); + + // Fail if the significance setup for the selected schemas are conflicting + if (uniqueSetups.size() > 1) { + var result = new Result(query); + result.hits().addError( + ErrorMessage.createIllegalQuery( + ("Inconsistent 'significance' configuration for the rank profile '%s' in the schemas %s. " + + "Use 'restrict' to limit the query to a subset of schemas " + + "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + + "Specify same 'significance' configuration for all selected schemas " + + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).") + .formatted(rankProfileName, perSchemaSetup.keySet()))); + return result; + } + + if (perSchemaSetup.isEmpty()) return execution.search(query); + var useSignificanceModel = uniqueSetups.iterator().next(); + if (!useSignificanceModel) return execution.search(query); + Language language = query.getModel().getParsingLanguage(); Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); @@ -72,6 +111,7 @@ public class SignificanceSearcher extends Searcher { public static double calculateIDF(long N, long nq_i) { return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5)); } + } diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 989fbb16973..086b47f5ae5 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -28,6 +28,7 @@ schema[].summaryclass[].fields[].dynamic bool default=false schema[].rankprofile[].name string schema[].rankprofile[].hasSummaryFeatures bool default=true schema[].rankprofile[].hasRankFeatures bool default=true +schema[].rankprofile[].significance.useModel bool default=false # The name of an input (query rank feature) accepted by this profile schema[].rankprofile[].input[].name string # The tensor type of an input (query rank feature) accepted by this profile |