diff options
Diffstat (limited to 'container-search/src')
39 files changed, 518 insertions, 186 deletions
diff --git a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java index f565315b775..4c39506ed96 100755 --- a/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java +++ b/container-search/src/main/java/ai/vespa/search/llm/LLMSearcher.java @@ -33,7 +33,7 @@ import java.util.stream.Collectors; @Beta public class LLMSearcher extends Searcher { - private static Logger log = Logger.getLogger(LLMSearcher.class.getName()); + private static final Logger log = Logger.getLogger(LLMSearcher.class.getName()); private static final String API_KEY_HEADER = "X-LLM-API-KEY"; private static final String STREAM_PROPERTY = "stream"; @@ -202,7 +202,7 @@ public class LLMSearcher extends Searcher { private static class TokenStats { - private long start; + private final long start; private long timeToFirstToken; private long timeToLastToken; private long tokens = 0; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java index 6a980c37f4e..b7d7b3d9ed5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/BoolField.java @@ -1,14 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Class representing a byte field in the result set - * - */ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; /** + * Class representing a byte field in the result set + * * @author bratseth */ public class BoolField extends DocsumField { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index 7f672582da3..02d970bf7c8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -28,8 +28,7 @@ public class DocsumDefinition { this.fields = documentSummary.fields().values() .stream() .map(field -> DocsumField.create(field.name(), field.type().asString())) - .collect(Collectors.toUnmodifiableMap(field -> field.getName(), - field -> field)); + .collect(Collectors.toUnmodifiableMap(DocsumField::getName, field -> field)); } public String name() { return name; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 7d258f71ebd..5e9c3dcc6ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -335,7 +335,7 @@ public abstract class AbstractParser implements CustomParser { /** * Segments a token * - * @param indexName the index name which preceeded this token, or null if none + * @param indexName the index name which preceded this token, or null if none * @param token the token to segment * @param quoted whether this segment is within quoted text * @return the resulting item diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index ea0cd2312a6..d3e6241a6e5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -134,10 +134,8 @@ abstract class SimpleParser extends StructuredParser { if (topLevelItem != null && topLevelItem != not) { // => neutral rank items becomes implicit positives not.addPositiveItem(getItemAsPositiveItem(topLevelItem, not)); - return not; - } else { - return not; } + return not; } if (topLevelItem != null) { return topLevelItem; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index f27e9bbc281..0ea9cefc63e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -109,14 +109,17 @@ public class StemmingSearcher extends Searcher { private Item replaceTerms(Query q, IndexFacts.Session indexFacts) { Language language = q.getModel().getParsingLanguage(); - if (language == Language.UNKNOWN) return q.getModel().getQueryTree().getRoot(); + if (language == Language.UNKNOWN) { + q.trace("Language is unknown, not stemming", 3); + return q.getModel().getQueryTree().getRoot(); + } StemContext context = new StemContext(); context.isCJK = language.isCjk(); context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); - q.trace("Stemming with language="+language, 3); + q.trace("Stemming with language " + language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 4ec3fa358d2..e01d03e96a7 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -605,7 +605,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setHits(int hits) { if (hits < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'hits' must be a positive number, not " + hits); this.hits = hits; } @@ -614,12 +614,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public void setOffset(int offset) { if (offset < 0) - throw new IllegalArgumentException("Must be a positive number"); + throw new IllegalArgumentException("'offset' must be a positive number, not " + offset); this.offset = offset; } /** Convenience method to set both the offset and the number of hits to return */ - public void setWindow(int offset,int hits) { + public void setWindow(int offset, int hits) { setOffset(offset); setHits(hits); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index d7fad148c8c..bfcf0af325d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -79,7 +79,7 @@ public abstract class InvokerFactory { success.add(node); } } - if ( ! cluster.isPartialGroupCoverageSufficient(success) && !acceptIncompleteCoverage) { + if ( ! cluster.isPartialGroupCoverageSufficient(group.hasSufficientCoverage(), success) && !acceptIncompleteCoverage) { return Optional.empty(); } if (invokers.isEmpty()) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 965ce4aeb94..c7af37b3a26 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -23,7 +23,7 @@ public class Group { // Using volatile to ensure visibility for reader. // All updates are done in a single writer thread - private volatile boolean hasSufficientCoverage = true; + private volatile boolean hasSufficientCoverage = false; private volatile boolean hasFullCoverage = true; private volatile long activeDocuments = 0; private volatile long targetActiveDocuments = 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 56545a32831..8f83d8ef5ce 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -226,17 +226,20 @@ public class SearchCluster implements NodeManager<Node> { // With just one group sufficient coverage may not be the same as full coverage, as the // group will always be marked sufficient for use. updateSufficientCoverage(group, true); - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), group.activeDocuments()); - trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments()); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), group.activeDocuments(), group.activeDocuments()); + trackGroupCoverageChanges(group, sufficientCoverage, group.activeDocuments(), group.activeDocuments()); } private void pingIterationCompletedMultipleGroups(SearchGroupsImpl groups) { groups.groups().forEach(Group::aggregateNodeValues); - long medianDocuments = groups.medianDocumentsPerGroup(); + long medianDocuments = groups.medianDocumentCount(); + long maxDocuments = groups.maxDocumentCount(); for (Group group : groups.groups()) { - boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.activeDocuments(), medianDocuments); + boolean sufficientCoverage = groups.isGroupCoverageSufficient(group.hasSufficientCoverage(), + group.activeDocuments(), medianDocuments, maxDocuments); updateSufficientCoverage(group, sufficientCoverage); - trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments); + trackGroupCoverageChanges(group, sufficientCoverage, medianDocuments, maxDocuments); } } @@ -261,7 +264,7 @@ public class SearchCluster implements NodeManager<Node> { /** * Calculate whether a subset of nodes in a group has enough coverage */ - private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments) { + private void trackGroupCoverageChanges(Group group, boolean fullCoverage, long medianDocuments, long maxDocuments) { if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about. boolean changed = group.fullCoverageStatusChanged(fullCoverage); if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) { @@ -278,7 +281,7 @@ public class SearchCluster implements NodeManager<Node> { unresponsive.append('\n').append(node); } String message = "Cluster " + clusterId + ": " + group + " has reduced coverage: " + - "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Active documents: " + group.activeDocuments() + "/" + maxDocuments + ", " + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java index 85063b8ef57..0bb694f610e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroups.java @@ -13,21 +13,30 @@ import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toSet; /** - * Simple interface for groups and their nodes in the content cluster + * Simple interface for groups and their nodes in the content cluster. + * * @author baldersheim */ public interface SearchGroups { + Group get(int id); + Set<Integer> keys(); + Collection<Group> groups(); + default boolean isEmpty() { return size() == 0; } + default Set<Node> nodes() { return groups().stream().flatMap(group -> group.nodes().stream()) .sorted(comparingInt(Node::key)) .collect(toCollection(LinkedHashSet::new)); } + int size(); - boolean isPartialGroupCoverageSufficient(Collection<Node> nodes); + + boolean isPartialGroupCoverageSufficient(boolean currentCoverageSufficient, Collection<Node> nodes); + } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java index c49a140804c..6528c5d2ae4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchGroupsImpl.java @@ -7,14 +7,17 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +/** + * @author baldersheim + */ public class SearchGroupsImpl implements SearchGroups { private final Map<Integer, Group> groups; - private final double minActivedocsPercentage; + private final double minActiveDocsPercentage; - public SearchGroupsImpl(Map<Integer, Group> groups, double minActivedocsPercentage) { + public SearchGroupsImpl(Map<Integer, Group> groups, double minActiveDocsPercentage) { this.groups = Map.copyOf(groups); - this.minActivedocsPercentage = minActivedocsPercentage; + this.minActiveDocsPercentage = minActiveDocsPercentage; } @Override public Group get(int id) { return groups.get(id); } @@ -23,23 +26,38 @@ public class SearchGroupsImpl implements SearchGroups { @Override public int size() { return groups.size(); } @Override - public boolean isPartialGroupCoverageSufficient(Collection<Node> nodes) { - if (size() == 1) - return true; - long activeDocuments = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); - return isGroupCoverageSufficient(activeDocuments, medianDocumentsPerGroup()); + public boolean isPartialGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, Collection<Node> nodes) { + if (size() == 1) return true; + long groupDocumentCount = nodes.stream().mapToLong(Node::getActiveDocuments).sum(); + return isGroupCoverageSufficient(currentIsGroupCoverageSufficient, + groupDocumentCount, medianDocumentCount(), maxDocumentCount()); } - public boolean isGroupCoverageSufficient(long activeDocuments, long medianDocuments) { - if (medianDocuments <= 0) return true; - double documentCoverage = 100.0 * (double) activeDocuments / medianDocuments; - return documentCoverage >= minActivedocsPercentage; + public boolean isGroupCoverageSufficient(boolean currentIsGroupCoverageSufficient, + long groupDocumentCount, long medianDocumentCount, long maxDocumentCount) { + if (medianDocumentCount <= 0) return true; + if (currentIsGroupCoverageSufficient) { + // To take a group *out of* rotation, require that it has less active documents than the median. + // This avoids scenarios where incorrect accounting in a single group takes all other groups offline. + double documentCoverage = 100.0 * (double) groupDocumentCount / medianDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } + else { + // to put a group *in* rotation, require that it has as many documents as the largest group, + // to avoid taking groups in too early when the majority of the groups have just been added. + double documentCoverage = 100.0 * (double) groupDocumentCount / maxDocumentCount; + return documentCoverage >= minActiveDocsPercentage; + } } - public long medianDocumentsPerGroup() { + public long medianDocumentCount() { if (isEmpty()) return 0; double[] activeDocuments = groups().stream().mapToDouble(Group::activeDocuments).toArray(); return (long) Quantiles.median().computeInPlace(activeDocuments); } + public long maxDocumentCount() { + return (long)groups().stream().mapToDouble(Group::activeDocuments).max().orElse(0); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java index 01167be6b8b..fdedbdc2fd9 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java +++ b/container-search/src/main/java/com/yahoo/search/handler/Json2SingleLevelMap.java @@ -64,8 +64,8 @@ class Json2SingleLevelMap { } void parse(Map<String, String> map, String parent) throws IOException { - for (parser.nextToken(); parser.getCurrentToken() != JsonToken.END_OBJECT; parser.nextToken()) { - String fieldName = parent + parser.getCurrentName(); + for (parser.nextToken(); parser.currentToken() != JsonToken.END_OBJECT; parser.nextToken()) { + String fieldName = parent + parser.currentName(); JsonToken token = parser.nextToken(); if ((token == JsonToken.VALUE_STRING) || (token == JsonToken.VALUE_NUMBER_FLOAT) || @@ -89,9 +89,9 @@ class Json2SingleLevelMap { } private String skipChildren(JsonParser parser, byte [] input) throws IOException { - JsonLocation start = parser.getCurrentLocation(); + JsonLocation start = parser.currentLocation(); parser.skipChildren(); - JsonLocation end = parser.getCurrentLocation(); + JsonLocation end = parser.currentLocation(); int offset = (int)start.getByteOffset() - 1; return new String(input, offset, (int)(end.getByteOffset() - offset), StandardCharsets.UTF_8); } diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index b1dd5624d18..09de1a24ef9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -113,6 +113,8 @@ public class Ranking implements Cloneable { private SoftTimeout softTimeout = new SoftTimeout(); + private boolean useSignificance = false; + public Ranking(Query parent) { this.parent = parent; this.rankFeatures = new RankFeatures(this); @@ -217,6 +219,14 @@ public class Ranking implements Cloneable { /** Returns whether rank features should be dumped with the result of this query, default false */ public boolean getListFeatures() { return listFeatures; } + /** Set whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public void setUseSignificance(boolean useSignificance) { this.useSignificance = useSignificance; } + + /** Returns whether to use significance in ranking */ + @com.yahoo.api.annotations.Beta + public boolean getUseSignificance() { return useSignificance; } + /** Returns the match phase rank settings of this. This is never null. */ public MatchPhase getMatchPhase() { return matchPhase; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index c90612425fa..90d5e04d2b6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -256,26 +256,18 @@ public class SelectParser implements Parser { } private Item buildFunctionCall(String key, Inspector value) { - switch (key) { - case WAND: - return buildWand(key, value); - case WEIGHTED_SET: - return buildWeightedSet(key, value); - case DOT_PRODUCT: - return buildDotProduct(key, value); - case GEO_LOCATION: - return buildGeoLocation(key, value); - case NEAREST_NEIGHBOR: - return buildNearestNeighbor(key, value); - case PREDICATE: - return buildPredicate(key, value); - case RANK: - return buildRank(key, value); - case WEAK_AND: - return buildWeakAnd(key, value); - default: - throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); - } + return switch (key) { + case WAND -> buildWand(key, value); + case WEIGHTED_SET -> buildWeightedSet(key, value); + case DOT_PRODUCT -> buildDotProduct(key, value); + case GEO_LOCATION -> buildGeoLocation(key, value); + case NEAREST_NEIGHBOR -> buildNearestNeighbor(key, value); + case PREDICATE -> buildPredicate(key, value); + case RANK -> buildRank(key, value); + case WEAK_AND -> buildWeakAnd(key, value); + default -> + throw newUnexpectedArgumentException(key, DOT_PRODUCT, NEAREST_NEIGHBOR, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + }; } private void addItemsFromInspector(CompositeItem item, Inspector inspector){ @@ -312,15 +304,11 @@ public class SelectParser implements Parser { private HashMap<Integer, Inspector> childMap(Inspector inspector) { HashMap<Integer, Inspector> children = new HashMap<>(); if (inspector.type() == ARRAY){ - inspector.traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.traverse((ArrayTraverser) children::put); } else if (inspector.type() == OBJECT){ if (inspector.field("children").valid()){ - inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { - children.put(index, new_value); - }); + inspector.field("children").traverse((ArrayTraverser) children::put); } } return children; @@ -336,9 +324,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation) { HashMap<String, Inspector> attributes = new HashMap<>(); if (annotation.type() == OBJECT){ - annotation.traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + annotation.traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -346,9 +332,7 @@ public class SelectParser implements Parser { private HashMap<String, Inspector> getAnnotationMap(Inspector inspector) { HashMap<String, Inspector> attributes = new HashMap<>(); if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ - inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { - attributes.put(index, new_value); - }); + inspector.field("attributes").traverse((ObjectTraverser) attributes::put); } return attributes; } @@ -487,7 +471,6 @@ public class SelectParser implements Parser { return item; } - @SuppressWarnings("deprecation") private CompositeItem buildWeakAnd(String key, Inspector value) { WeakAndItem weakAnd = new WeakAndItem(); addItemsFromInspector(weakAnd, value); @@ -576,8 +559,7 @@ public class SelectParser implements Parser { } }); } - if (out instanceof IntItem && annotations != null) { - IntItem number = (IntItem) out; + if (out instanceof IntItem number && annotations != null) { Integer hitLimit = getCappedRangeSearchParameter(annotations); if (hitLimit != null) { number.setHitLimit(hitLimit); @@ -631,12 +613,13 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("The first array element under 'equals' should be a field name string " + "but was " + children.get(0)); String field = children.get(0).asString(); - switch (children.get(1).type()) { - case BOOL: return new BoolItem(children.get(1).asBool(), field); - case LONG: return new IntItem(children.get(1).asLong(), field); - default: throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + - "or int value but was " + children.get(1)); - } + return switch (children.get(1).type()) { + case BOOL -> new BoolItem(children.get(1).asBool(), field); + case LONG -> new IntItem(children.get(1).asLong(), field); + default -> + throw new IllegalArgumentException("The second array element under 'equals' should be a boolean " + + "or int value but was " + children.get(1)); + }; } private Item buildRange(String key, Inspector value) { @@ -661,15 +644,15 @@ public class SelectParser implements Parser { throw new IllegalArgumentException("Expected a numeric argument to range, but got the string '" + bound.asString() + "'"); } if (operator.equals("=")) { - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; equals[0] = true; } if (operator.equals(">=") || operator.equals(">")){ - bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[0] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[0] = operator; } else if (operator.equals("<=") || operator.equals("<")){ - bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + bounds[1] = (bound.type() == DOUBLE) ? (Number) bound.asDouble() : (Number) bound.asLong(); operators[1] = operator; } @@ -680,20 +663,13 @@ public class SelectParser implements Parser { } else if (operators[0] == null || operators[1] == null) { int index = (operators[0] == null) ? 1 : 0; - switch (operators[index]){ - case ">=": - range = buildGreaterThanOrEquals(field, bounds[index].toString()); - break; - case ">": - range = buildGreaterThan(field, bounds[index].toString()); - break; - case "<": - range = buildLessThan(field, bounds[index].toString()); - break; - case "<=": - range = buildLessThanOrEquals(field, bounds[index].toString()); - break; - } + range = switch (operators[index]) { + case ">=" -> buildGreaterThanOrEquals(field, bounds[index].toString()); + case ">" -> buildGreaterThan(field, bounds[index].toString()); + case "<" -> buildLessThan(field, bounds[index].toString()); + case "<=" -> buildLessThanOrEquals(field, bounds[index].toString()); + default -> range; + }; } else { range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<")); @@ -890,7 +866,7 @@ public class SelectParser implements Parser { String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; if (FUNCTION_CALLS.contains(key)) { return instantiateCompositeLeaf(field, key, value); - } else if ( ! possibleLeafFunctionName.equals("")){ + } else if (!possibleLeafFunctionName.isEmpty()){ return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); } else { return instantiateWordItem(field, key, value); @@ -898,24 +874,16 @@ public class SelectParser implements Parser { } private Item instantiateCompositeLeaf(String field, String key, Inspector value) { - switch (key) { - case SAME_ELEMENT: - return instantiateSameElementItem(field, key, value); - case PHRASE: - return instantiatePhraseItem(field, key, value); - case NEAR: - return instantiateNearItem(field, key, value); - case ONEAR: - return instantiateONearItem(field, key, value); - case EQUIV: - return instantiateEquivItem(field, key, value); - case FUZZY: - return instantiateFuzzyItem(field, key, value); - case ALTERNATIVES: - return instantiateWordAlternativesItem(field, key, value); - default: - throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); - } + return switch (key) { + case SAME_ELEMENT -> instantiateSameElementItem(field, key, value); + case PHRASE -> instantiatePhraseItem(field, key, value); + case NEAR -> instantiateNearItem(field, key, value); + case ONEAR -> instantiateONearItem(field, key, value); + case EQUIV -> instantiateEquivItem(field, key, value); + case FUZZY -> instantiateFuzzyItem(field, key, value); + case ALTERNATIVES -> instantiateWordAlternativesItem(field, key, value); + default -> throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); + }; } private Item instantiateWordItem(String field, String key, Inspector value) { @@ -944,8 +912,8 @@ public class SelectParser implements Parser { Preconditions.checkArgument((prefixMatch ? 1 : 0) + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, "Only one of prefix, substring and suffix can be set."); - final TaggableItem wordItem; + WordItem wordItem; if (exactMatch) { wordItem = new ExactStringItem(wordData, fromQuery); } else if (prefixMatch) { @@ -958,13 +926,11 @@ public class SelectParser implements Parser { wordItem = new WordItem(wordData, fromQuery); } - if (wordItem instanceof WordItem) { - prepareWord(field, value, (WordItem) wordItem); - } + prepareWord(field, value, wordItem); if (language != Language.ENGLISH) - ((Item)wordItem).setLanguage(language); + wordItem.setLanguage(language); - return (Item) leafStyleSettings(getAnnotations(value), wordItem); + return leafStyleSettings(getAnnotations(value), wordItem); } private Language decideParsingLanguage(Inspector value, String wordData) { @@ -974,9 +940,8 @@ public class SelectParser implements Parser { if (language != Language.UNKNOWN) return language; Optional<Language> explicitLanguage = query.getExplicitLanguage(); - if (explicitLanguage.isPresent()) return explicitLanguage.get(); + return explicitLanguage.orElse(Language.ENGLISH); - return Language.ENGLISH; } private void prepareWord(String field, Inspector value, WordItem wordItem) { @@ -1094,7 +1059,7 @@ public class SelectParser implements Parser { Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); if (distance != null) { - near.setDistance((int)distance); + near.setDistance(distance); } return near; } @@ -1120,7 +1085,8 @@ public class SelectParser implements Parser { private Item instantiateEquivItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size()); + Preconditions.checkArgument(children.size() >= 2, + "Expected 2 or more arguments, got %s.", children.size()); EquivItem equiv = new EquivItem(); equiv.setIndexName(field); @@ -1159,8 +1125,9 @@ public class SelectParser implements Parser { private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); - Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); - Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type()); + Preconditions.checkArgument(!children.isEmpty(), "Expected 1 or more arguments, got none"); + Preconditions.checkArgument(children.get(0).type() == OBJECT, + "Expected OBJECT, got %s.", children.get(0).type()); List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 7536e74042c..72a1a7d3430 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -12,6 +12,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.yql.MinimalQueryInserter; import com.yahoo.yolean.chain.After; +import com.yahoo.yolean.chain.Provides; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -19,10 +20,12 @@ import com.yahoo.yolean.chain.After; * * @author karowan */ +@Provides(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) @After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { + public static final String REPLACE_OR_WITH_WEAKAND = "replace-or-with-weakand"; static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace"); - static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); + public static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); @Override public Result search(Query query, Execution execution) { if (!query.properties().getBoolean(WEAKAND_REPLACE)) { diff --git a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java index a5b8d328a7a..9583e9885e7 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java +++ b/container-search/src/main/java/com/yahoo/search/schema/RankProfile.java @@ -36,6 +36,7 @@ public class RankProfile { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<String, InputType> inputs; // Assigned when this is added to a schema @@ -45,6 +46,7 @@ public class RankProfile { this.name = builder.name; this.hasSummaryFeatures = builder.hasSummaryFeatures; this.hasRankFeatures = builder.hasRankFeatures; + this.useSignificanceModel = builder.useSignificanceModel; this.inputs = Collections.unmodifiableMap(builder.inputs); } @@ -66,6 +68,9 @@ public class RankProfile { /** Returns true if this rank profile has rank features. */ public boolean hasRankFeatures() { return hasRankFeatures; } + /** Returns true if this rank profile should use significance models. */ + public boolean useSignificanceModel() { return useSignificanceModel; } + /** Returns the inputs explicitly declared in this rank profile. */ public Map<String, InputType> inputs() { return inputs; } @@ -76,13 +81,14 @@ public class RankProfile { if ( ! other.name.equals(this.name)) return false; if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; if ( other.hasRankFeatures != this.hasRankFeatures) return false; + if ( other.useSignificanceModel != this.useSignificanceModel) return false; if ( ! other.inputs.equals(this.inputs)) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); + return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, useSignificanceModel, inputs); } @Override @@ -95,6 +101,7 @@ public class RankProfile { private final String name; private boolean hasSummaryFeatures = true; private boolean hasRankFeatures = true; + private boolean useSignificanceModel = false; private final Map<String, InputType> inputs = new LinkedHashMap<>(); public Builder(String name) { @@ -116,6 +123,8 @@ public class RankProfile { return this; } + public Builder setUseSignificanceModel(boolean use) { this.useSignificanceModel = use; return this; } + public RankProfile build() { return new RankProfile(this); } diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java index d28c2db2b9e..77f27d3d411 100644 --- a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java @@ -22,9 +22,10 @@ class SchemaInfoConfigurer { Schema.Builder builder = new Schema.Builder(schemaInfoConfig.name()); for (var profileConfig : schemaInfoConfig.rankprofile()) { - RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); - profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); - profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); + RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()) + .setHasSummaryFeatures(profileConfig.hasSummaryFeatures()) + .setHasRankFeatures(profileConfig.hasRankFeatures()) + .setUseSignificanceModel(profileConfig.significance().useModel()); for (var inputConfig : profileConfig.input()) profileBuilder.addInput(inputConfig.name(), RankProfile.InputType.fromSpec(inputConfig.type())); builder.add(profileBuilder.build()); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java index 69a1f8ec6cb..c03a74ea2c5 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/VespaSearchers.java @@ -34,7 +34,8 @@ public class VespaSearchers { com.yahoo.prelude.searcher.PosSearcher.class, com.yahoo.prelude.semantics.SemanticSearcher.class, com.yahoo.search.grouping.GroupingQueryParser.class, - com.yahoo.search.querytransform.WeakAndReplacementSearcher.class); + com.yahoo.search.querytransform.WeakAndReplacementSearcher.class, + com.yahoo.search.searchers.OpportunisticWeakAndSearcher.class); public static final Collection<ChainedComponentModel> nativeSearcherModels; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java new file mode 100644 index 00000000000..c512e821128 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcher.java @@ -0,0 +1,85 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.searchers; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.WeakAndReplacementSearcher; +import com.yahoo.search.searchchain.Execution; + +import static com.yahoo.search.querytransform.WeakAndReplacementSearcher.WAND_HITS; + +/** + * Will opportunistically replace the WeakAND with an AND as it is faster. + * If enough hits are returned all is good and we return. If not we fall back to the original query. + * + * @author baldersheim + */ +@Beta +@After(WeakAndReplacementSearcher.REPLACE_OR_WITH_WEAKAND) +public class OpportunisticWeakAndSearcher extends Searcher { + static final CompoundName OPPORTUNISTIC_AND = CompoundName.from("weakAnd.opportunistic.and"); + + @Override + public Result search(Query query, Execution execution) { + if (!query.properties().getBoolean(OPPORTUNISTIC_AND)) { + return execution.search(query); + } + + Item originalRoot = query.getModel().getQueryTree().getRoot(); + int targetHits = targetHits(originalRoot); + if (targetHits >= 0) { + query.getModel().getQueryTree().setRoot(weakAnd2AndRecurse(originalRoot.clone())); + query.trace("WeakAND => AND", true, 2); + Result result = execution.search(query); + if (result.getHitCount() >= query.properties().getInteger(WAND_HITS)) { + return result; + } + query.getModel().getQueryTree().setRoot(originalRoot); + return execution.search(query); + } + return execution.search(query); + } + + // returns targetHits for the first WeakAndItem found, -1 if none found. + static int targetHits(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return -1; + if (item instanceof WeakAndItem weakAndItem) return weakAndItem.getN(); + for (int i = 0; i < compositeItem.getItemCount(); i++) { + int targetHits = targetHits(compositeItem.getItem(i)); + if (targetHits >= 0) return targetHits; + } + return -1; + } + + static Item weakAnd2AndRecurse(Item item) { + if (!(item instanceof CompositeItem compositeItem)) return item; + compositeItem = weakAnd2And(compositeItem); + for (int i = 0; i < compositeItem.getItemCount(); i++) { + Item subItem = compositeItem.getItem(i); + Item replacedItem = weakAnd2AndRecurse(subItem); + if (replacedItem != subItem) { + compositeItem.setItem(i, replacedItem); + } + } + return compositeItem; + } + + private static CompositeItem weakAnd2And(CompositeItem item) { + if (item instanceof WeakAndItem weakAndItem) { + AndItem andItem = new AndItem(); + andItem.setWeight(weakAndItem.getWeight()); + item.items().forEach(andItem::addItem); + return andItem; + } + return item; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index 0a42bf8a259..e3a559da8f9 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -14,33 +14,72 @@ import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; +import java.util.HashSet; import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; /** * Sets significance values on word items in the query tree. * - * @author MariusArhaug + * @author Marius Arhaug */ - @Provides(SignificanceSearcher.SIGNIFICANCE) @Before(STEMMING) public class SignificanceSearcher extends Searcher { public final static String SIGNIFICANCE = "Significance"; - private final SignificanceModelRegistry significanceModelRegistry; + private static final Logger log = Logger.getLogger(SignificanceSearcher.class.getName()); + + private final SignificanceModelRegistry significanceModelRegistry; + private final SchemaInfo schemaInfo; @Inject - public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry) { + public SignificanceSearcher(SignificanceModelRegistry significanceModelRegistry, SchemaInfo schemaInfo) { this.significanceModelRegistry = significanceModelRegistry; + this.schemaInfo = schemaInfo; } @Override public Result search(Query query, Execution execution) { + var rankProfileName = query.getRanking().getProfile(); + + // Determine significance setup per schema for the given rank profile + var perSchemaSetup = schemaInfo.newSession(query).schemas().stream() + .collect(Collectors.toMap(Schema::name, schema -> + // Fallback to disabled if the rank profile is not found in the schema + // This will result in a failure later (in a "backend searcher") anyway. + Optional.ofNullable(schema.rankProfiles().get(rankProfileName)) + .map(RankProfile::useSignificanceModel).orElse(false))); + var uniqueSetups = new HashSet<>(perSchemaSetup.values()); + + // Fail if the significance setup for the selected schemas are conflicting + if (uniqueSetups.size() > 1) { + var result = new Result(query); + result.hits().addError( + ErrorMessage.createIllegalQuery( + ("Inconsistent 'significance' configuration for the rank profile '%s' in the schemas %s. " + + "Use 'restrict' to limit the query to a subset of schemas " + + "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + + "Specify same 'significance' configuration for all selected schemas " + + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).") + .formatted(rankProfileName, perSchemaSetup.keySet()))); + return result; + } + + if (perSchemaSetup.isEmpty()) return execution.search(query); + var useSignificanceModel = uniqueSetups.iterator().next(); + if (!useSignificanceModel) return execution.search(query); + Language language = query.getModel().getParsingLanguage(); Optional<SignificanceModel> model = significanceModelRegistry.getModel(language); @@ -72,6 +111,7 @@ public class SignificanceSearcher extends Searcher { public static double calculateIDF(long N, long nq_i) { return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5)); } + } diff --git a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def index 989fbb16973..086b47f5ae5 100644 --- a/container-search/src/main/resources/configdefinitions/container.search.schema-info.def +++ b/container-search/src/main/resources/configdefinitions/container.search.schema-info.def @@ -28,6 +28,7 @@ schema[].summaryclass[].fields[].dynamic bool default=false schema[].rankprofile[].name string schema[].rankprofile[].hasSummaryFeatures bool default=true schema[].rankprofile[].hasRankFeatures bool default=true +schema[].rankprofile[].significance.useModel bool default=false # The name of an input (query rank feature) accepted by this profile schema[].rankprofile[].input[].name string # The tensor type of an input (query rank feature) accepted by this profile diff --git a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java index 3baa9715c34..2cc72a43f43 100755 --- a/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java +++ b/container-search/src/test/java/ai/vespa/search/llm/LLMSearcherTest.java @@ -13,6 +13,7 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.result.EventStream; import com.yahoo.search.searchchain.Execution; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.net.URLEncoder; @@ -127,6 +128,7 @@ public class LLMSearcherTest { } @Test + @Disabled public void testAsyncGeneration() { var executor = Executors.newFixedThreadPool(2); var sb = new StringBuilder(); diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index 0516a8a227a..a662c445bc7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -22,6 +22,15 @@ public class QueryCanonicalizerTestCase { } @Test + void testNoCanonicalizationWithWhereTrue() { + CompositeItem root = new AndItem(); + + root.addItem(new TrueItem()); + root.addItem(new WordItem("word")); + assertCanonicalized("AND TRUE word", null, root); + } + + @Test void testSingleLevelSingleItemNonReducibleComposite() { CompositeItem root = new WeakAndItem(); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 8ced5c81372..12110e2d2f3 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -220,7 +220,7 @@ public class QueryTestCase { void testNegativeHitValue() { assertQueryError( "?query=test&hits=-1", - "Could not set 'hits' to '-1': Must be a positive number"); + "Could not set 'hits' to '-1': 'hits' must be a positive number, not -1"); } @Test @@ -241,7 +241,7 @@ public class QueryTestCase { void testNegativeOffsetValue() { assertQueryError( "?query=test&offset=-1", - "Could not set 'offset' to '-1': Must be a positive number"); + "Could not set 'offset' to '-1': 'offset' must be a positive number, not -1"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java index 2a9eaa86674..e7085b093f3 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java @@ -48,6 +48,19 @@ public class SearchClusterCoverageTest { } @Test + void three_groups_of_which_two_were_just_added() { + var tester = new SearchClusterTester(3, 3); + + tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(80, 1); + tester.setDocsPerNode(80, 2); + tester.pingIterationCompleted(); + assertTrue(tester.group(0).hasSufficientCoverage()); + assertFalse(tester.group(1).hasSufficientCoverage()); + assertFalse(tester.group(2).hasSufficientCoverage()); + } + + @Test void three_groups_one_missing_docs_but_too_few() { var tester = new SearchClusterTester(3, 3); @@ -65,6 +78,10 @@ public class SearchClusterCoverageTest { var tester = new SearchClusterTester(3, 3); tester.setDocsPerNode(100, 0); + tester.setDocsPerNode(100, 1); + tester.setDocsPerNode(100, 2); + tester.pingIterationCompleted(); + tester.setDocsPerNode(100, 0); tester.setDocsPerNode(150, 1); tester.setDocsPerNode(100, 2); tester.pingIterationCompleted(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 1b36c2b8151..8ac4f067876 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -200,8 +200,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusIsDefaultDownWithLocalDispatch() { try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "b")) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -211,8 +209,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusStaysUpWithLocalDispatchAndClusterSize1() { try (State test = new State("cluster.1", 1, HostName.getLocalhost())) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); @@ -225,8 +221,6 @@ public class SearchClusterTest { @Test void requireThatVipStatusIsDefaultDownWithLocalDispatchAndClusterSize2() { try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "otherhost")) { - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - assertFalse(test.vipStatus.isInRotation()); test.waitOneFullPingRound(); assertTrue(test.vipStatus.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java index 52f5fd0cafb..7b91a5d3c25 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WeakAndReplacementSearcherTestCase.java @@ -23,6 +23,7 @@ import static com.yahoo.search.querytransform.WeakAndReplacementSearcher.WAND_HI import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; public class WeakAndReplacementSearcherTestCase { @@ -57,7 +58,7 @@ public class WeakAndReplacementSearcherTestCase { Result result = buildExec().search(query); Item root = TestUtils.getQueryTreeRoot(result); assertFalse(orItemsExist(root)); - assertTrue(root instanceof WeakAndItem); + assertInstanceOf(WeakAndItem.class, root); assertEquals(N, ((WeakAndItem) root).getN()); } @@ -103,24 +104,22 @@ public class WeakAndReplacementSearcherTestCase { if (item1 != item2) { return false; } - if (!(item1 instanceof CompositeItem)) { + if (!(item1 instanceof CompositeItem compositeItem1)) { return true; } - CompositeItem compositeItem1 = (CompositeItem) item1; CompositeItem compositeItem2 = (CompositeItem) item2; return IntStream.range(0, compositeItem1.getItemCount()) .allMatch(i -> deepEquals(compositeItem1.getItem(i), compositeItem2.getItem(i))); } private boolean orItemsExist(Item item) { - if (!(item instanceof CompositeItem)) { + if (!(item instanceof CompositeItem compositeItem)) { return false; } if (item instanceof OrItem) { return true; } - CompositeItem compositeItem = (CompositeItem) item; return compositeItem.items().stream().anyMatch(this::orItemsExist); } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java index c0a677b2094..2cfb6552379 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/EventRendererTestCase.java @@ -141,7 +141,7 @@ public class EventRendererTestCase { }); assertFalse(future.isDone()); result = render(new Result(new Query(), newHitGroup(tokenStream, "token_stream"))); - assertTrue(future.isDone()); // Renderer waits for async completion + future.join(); // Renderer waits for async completion } finally { executor.shutdownNow(); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index ffa6c82e941..611df6ad284 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1532,7 +1532,7 @@ public class JsonRendererTestCase { + "}"; assertEquals( "Unexpected character ('a' (code 97)): was expecting comma to separate Object entries\n" + - " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 41]", + " at [Source: (String)\"{ \"root\": { \"invalidvalue\": 1adsf, }}\"; line: 1, column: 40]", validateJSON(json)); } diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java index 3e98b911fc8..2ba399cf42d 100644 --- a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java +++ b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java @@ -77,6 +77,7 @@ public class SchemaInfoTester { .addInput("query(myTensor1)", InputType.fromSpec("tensor(x[10])")) .build()) .add(new RankProfile.Builder("bOnly") + .setUseSignificanceModel(true) .addInput("query(myTensor1)", InputType.fromSpec("tensor(a{},b{})")) .build()) .build()); @@ -129,7 +130,8 @@ public class SchemaInfoTester { rankProfileInconsistentB.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); schemaB.rankprofile(rankProfileInconsistentB); var rankProfileBOnly = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileBOnly.name("bOnly"); + rankProfileBOnly.name("bOnly") + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder().useModel(true)); rankProfileBOnly.input(new SchemaInfoConfig.Schema.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); schemaB.rankprofile(rankProfileBOnly); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java new file mode 100644 index 00000000000..c099cf437f8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/OpportunisticWeakAndSearcherTestCase.java @@ -0,0 +1,42 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.searchers; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class OpportunisticWeakAndSearcherTestCase { + private static Item buildQueryItem(CompositeItem root, CompositeItem injectAtLevel2) { + root.addItem(new WordItem("text")); + injectAtLevel2.addItem(new WordItem("a")); + injectAtLevel2.addItem(new WordItem("b")); + root.addItem(injectAtLevel2); + return root; + } + + @Test + public void requireThatWeakAndIsDetected() { + assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(new OrItem())); + assertEquals(33, OpportunisticWeakAndSearcher.targetHits(new WeakAndItem(33))); + assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new WeakAndItem(77)))); + assertEquals(77, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new AndItem(), new WeakAndItem(77)))); + assertEquals(-1, OpportunisticWeakAndSearcher.targetHits(buildQueryItem(new OrItem(), new AndItem()))); + } + + @Test + public void requireThatWeakAndIsReplacedWithAnd() { + assertEquals(buildQueryItem(new OrItem(), new AndItem()), + OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new OrItem(), new WeakAndItem()))); + assertEquals(buildQueryItem(new AndItem(), new AndItem()), + OpportunisticWeakAndSearcher.weakAnd2AndRecurse(buildQueryItem(new AndItem(), new WeakAndItem()))); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json b/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json new file mode 100644 index 00000000000..d1fcf5bd411 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/model/docv1.json @@ -0,0 +1,18 @@ +{ + "version" : "1.0", + "id" : "test::1", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 12, + "language" : "en", + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2 + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json b/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json new file mode 100644 index 00000000000..7c751544803 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/significance/model/docv2.json @@ -0,0 +1,31 @@ +{ + "version" : "2.0", + "id" : "test::2", + "description" : "desc", + "languages" : { + "en": { + "description" : "english model", + "document-count" : 16, + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 4, + "additional": 2 + } + }, + "nb": { + "description" : "norwegian model", + "document-count" : 20, + "document-frequencies" : { + "usa" : 2, + "hello": 10, + "verden": 5, + "test": 2, + "norge": 11, + "ja": 12, + "nei": 3 + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/significance/model/en.json b/container-search/src/test/java/com/yahoo/search/significance/model/en.json index 50bae5e3451..a0acd1747ce 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/model/en.json +++ b/container-search/src/test/java/com/yahoo/search/significance/model/en.json @@ -2,13 +2,18 @@ "version" : "1.0", "id" : "test::1", "description" : "desc", - "corpus-size" : 10, - "language" : "en", - "word-count" : 4, - "frequencies" : { - "usa" : 2, - "hello": 3, - "world": 5, - "test": 2 + "languages" : { + "en": { + "description" : "english model", + "document-count" : 16, + "language" : "en", + "document-frequencies" : { + "usa" : 2, + "hello": 3, + "world": 5, + "test": 2, + "additional": 2 + } + } } } diff --git a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java index 890db3abb51..eaa66755608 100644 --- a/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/significance/test/SignificanceSearcherTest.java @@ -2,6 +2,7 @@ package com.yahoo.search.significance.test; import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.language.Language; import com.yahoo.language.significance.SignificanceModel; import com.yahoo.language.significance.SignificanceModelRegistry; @@ -10,12 +11,19 @@ import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.schema.DocumentSummary; +import com.yahoo.search.schema.RankProfile; +import com.yahoo.search.schema.Schema; +import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.significance.SignificanceSearcher; +import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.jupiter.api.Test; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import static com.yahoo.test.JunitCompat.assertEquals; @@ -29,12 +37,18 @@ public class SignificanceSearcherTest { SignificanceModelRegistry significanceModelRegistry; SignificanceSearcher searcher; - public SignificanceSearcherTest() { - HashMap<Language, Path> map = new HashMap<>(); - map.put(Language.ENGLISH, Path.of("src/test/java/com/yahoo/search/significance/model/en.json")); - significanceModelRegistry = new DefaultSignificanceModelRegistry(map); - searcher = new SignificanceSearcher(significanceModelRegistry); + public SignificanceSearcherTest() { + List<Path> models = new ArrayList<>(); + models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv1.json")); + models.add(Path.of("src/test/java/com/yahoo/search/significance/model/docv2.json")); + var schema = new Schema.Builder("music") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(true) + .build()); + significanceModelRegistry = new DefaultSignificanceModelRegistry(models); + searcher = new SignificanceSearcher(significanceModelRegistry, new SchemaInfo(List.of(schema.build()), List.of())); } private Execution createExecution(SignificanceSearcher searcher) { @@ -47,8 +61,29 @@ public class SignificanceSearcherTest { @Test void testSignificanceValueOnSimpleQuery() { + Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("hello", true); + root.addItem(tmp); + + q.getModel().getQueryTree().setRoot(root); + + SignificanceModel model = significanceModelRegistry.getModel(Language.ENGLISH).get(); + var helloFrequency = model.documentFrequency("hello"); + var helloSignificanceValue = SignificanceSearcher.calculateIDF(helloFrequency.corpusSize(), helloFrequency.frequency()); + Result r = createExecution(searcher).search(q); + + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + assertEquals(helloSignificanceValue, w0.getSignificance()); + } + @Test + void testSignificanceValueOnSimpleANDQuery() { Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); AndItem root = new AndItem(); WordItem tmp; tmp = new WordItem("Hello", true); @@ -79,6 +114,7 @@ public class SignificanceSearcherTest { @Test void testSignificanceValueOnRecursiveQuery() { Query q = new Query(); + q.getRanking().setProfile("significance-ranking"); AndItem root = new AndItem(); WordItem child1 = new WordItem("hello", true); @@ -117,7 +153,7 @@ public class SignificanceSearcherTest { assertEquals(helloSignificanceValue, w0.getSignificance()); assertEquals(testSignificanceValue, w1.getSignificance()); - assertEquals(SignificanceSearcher.calculateIDF(10, 2), w3.getSignificance()); + assertEquals(SignificanceSearcher.calculateIDF(16, 2), w3.getSignificance()); } @@ -150,4 +186,36 @@ public class SignificanceSearcherTest { assertEquals(w0_1.getSignificance(), w1.getSignificance()); } + + @Test + public void failsOnConflictingSignificanceConfiguration() { + var musicSchema = new Schema.Builder("music") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(true) + .build()) + .build(); + var albumSchema = new Schema.Builder("album") + .add(new DocumentSummary.Builder("default").build()) + .add(new RankProfile.Builder("significance-ranking") + .setUseSignificanceModel(false) + .build()) + .build(); + var searcher = new SignificanceSearcher( + significanceModelRegistry, new SchemaInfo(List.of(musicSchema, albumSchema), List.of())); + + var query = new Query(); + query.getRanking().setProfile("significance-ranking"); + + var result = createExecution(searcher).search(query); + assertEquals(1, result.hits().getErrorHit().errors().size()); + + var errorMessage = result.hits().getError(); + assertEquals("Inconsistent 'significance' configuration for the rank profile 'significance-ranking' in the schemas [music, album]. " + + "Use 'restrict' to limit the query to a subset of schemas " + + "(https://docs.vespa.ai/en/schemas.html#multiple-schemas). " + + "Specify same 'significance' configuration for all selected schemas " + + "(https://docs.vespa.ai/en/reference/schema-reference.html#significance).", + errorMessage.getDetailedMessage()); + } } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 91f5984481a..1ba10a31638 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -17,6 +17,7 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NumericInItem; +import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; @@ -30,6 +31,7 @@ import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; @@ -557,8 +559,7 @@ public class YqlParserTestCase { "title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " + "and title contains ({id: 2}\"saint\") " + "and title contains ({id: 3}\"angel\")"); - assertEquals("AND title:madonna title:saint title:angel", - parsed.toString()); + assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); AndItem root = (AndItem) parsed.getRoot(); WordItem first = (WordItem) root.getItem(0); WordItem second = (WordItem) root.getItem(1); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index f863816dab2..b15663e0ce6 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -56,6 +56,11 @@ public class SelectTestCase { //------------------------------------------------------------------- "where" tests @Test + void testSimple() { + assertParse("{'contains' : ['title', 'madonna']}", "title:madonna"); + } + + @Test void test_contains() { ObjectNode json = jsonMapper.createObjectNode(); ArrayNode arrayNode = jsonMapper.createArrayNode(); @@ -65,16 +70,9 @@ public class SelectTestCase { } @Test - void test() { - assertParse("{'contains' : ['title', 'madonna']}", - "title:madonna"); - } - - - @Test void testDottedFieldNames() { assertParse("{ 'contains' : ['my.nested.title', 'madonna']}", - "my.nested.title:madonna"); + "my.nested.title:madonna"); } @Test @@ -360,12 +358,12 @@ public class SelectTestCase { @Test void testRaw() { Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot(); - assertTrue(root instanceof WordItem); + assertInstanceOf(WordItem.class, root); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem) root).getWord()); root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot(); - assertTrue(root instanceof WordItem); + assertInstanceOf(WordItem.class, root); assertFalse(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem) root).getWord()); } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java index cd9ef708920..25b54267242 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java @@ -167,6 +167,7 @@ public class StreamingSearcherTestCase { Query[] queries = new Query[4]; // Increase coverage for (int i = 0; i<queries.length; i++) { Query query = new Query(queryString); + query.setTimeout(1000); if (i == 0) { } else if (i == 1) { query.getPresentation().setSummary("summary"); |